complete dualis async

fetchDUALIS.py

@@ -43,7 +43,7 @@ async def checkUser(email: str, password: str):
         return token, cookie
 
 
-def getKurs(token: int, cookie: str):
+async def getKurs(token: int, cookie: str):
     """
     Bestimmt aus der ersten Prüfung den Kursbezeichner des Users.
     TODO: Umstellen auf Bezeichner INKL. Standort
@@ -54,34 +54,34 @@ def getKurs(token: int, cookie: str):
     try:
         headers["Cookie"] = "cnsc=" + cookie
         token = str(token)
-        response = requests.request("GET", url +
+        async with httpx.AsyncClient as s:
-                                    "?APPNAME=CampusNet&PRGNAME=COURSERESULTS&ARGUMENTS=-N" + token + ",-N000307,",
+            response = await s.get(url=f"{url}?APPNAME=CampusNet&PRGNAME=COURSERESULTS&ARGUMENTS=-N{token},-N000307,",
-                                    headers=headers, data={})
+                                   headers=headers)
-        html = BeautifulSoup(response.text, 'lxml')
+            html = BeautifulSoup(response.text, 'lxml')
-        link = html.body.find('a', attrs={'id': "Popup_details0001"})['href']
+            link = html.body.find('a', attrs={'id': "Popup_details0001"})['href']
-        response = requests.request("GET", url + link[21:], headers=headers, data={})
+            response = requests.request("GET", url + link[21:], headers=headers, data={})
-        html = BeautifulSoup(response.text, 'lxml')
+            html = BeautifulSoup(response.text, 'lxml')
-        content = html.body.find('td', attrs={'class': 'level02'}).text
+            content = html.body.find('td', attrs={'class': 'level02'}).text
-        start = content.find(" ") + 4
+            start = content.find(" ") + 4
-        end = start + (content[start:].find(" "))
+            end = start + (content[start:].find(" "))
-        kurs = content[start:end]
+            kurs = content[start:end]
     except AttributeError:
         kurs = 0
     return kurs
 
 
-def logOut(token: int, cookie: str):
+async def logOut(token: int, cookie: str):
     """
     Invalidiert Token und Cookie bei Dualis.
     :param token:
     :param cookie:
     """
     headers["Cookie"] = "cnsc=" + cookie
-    requests.request("GET", url + "?APPNAME=CampusNet&PRGNAME=LOGOUT&ARGUMENTS=-N" + str(token)
+    async with httpx.AsyncClient() as s:
-                     + ", -N001", headers=headers, data={})
+        await s.get(url=f"{url}?APPNAME=CampusNet&PRGNAME=LOGOUT&ARGUMENTS=-N{token}, -N001", headers=headers)
 
 
-def getSem(token: int, cookie: str):
+async def getSem(token: int, cookie: str):
     """
     Liefert die Liste aller auf Dualis verfügbaren Semester.
     :param token:
@@ -90,21 +90,21 @@ def getSem(token: int, cookie: str):
     """
     headers["Cookie"] = "cnsc=" + cookie
     token = str(token)
-    response = requests.request("GET", url +
+    async with httpx.AsyncClient() as s:
-                                "?APPNAME=CampusNet&PRGNAME=COURSERESULTS&ARGUMENTS=-N" + token + ",-N000307,",
+        response = await s.get(url=f"{url}?APPNAME=CampusNet&PRGNAME=COURSERESULTS&ARGUMENTS=-N{token},-N000307,",
-                                headers=headers, data={})
+                               headers=headers)
-    html = BeautifulSoup(response.text, 'lxml')
+        html = BeautifulSoup(response.text, 'lxml')
-    select = html.find('select')
+        select = html.find('select')
-    select = select.find_all(value=True)
+        select = select.find_all(value=True)
-    optlist = []
+        optlist = []
-    for i in select:
+        for i in select:
-        t = i.text.replace("Wi", "Winter").replace("So", "Sommer")
+            t = i.text.replace("Wi", "Winter").replace("So", "Sommer")
-        t = t.replace("Se", "semester")
+            t = t.replace("Se", "semester")
-        optlist += [[t, i['value']]]
+            optlist += [[t, i['value']]]
-    return optlist
+        return optlist
 
 
-def getResults(token, cookie: str, resl: str):
+async def getResults(token, cookie: str, resl: str):
     """
     Liefert die Liste aller Prüfungsergebnisse eines Semesters.
     :param token:
@@ -113,31 +113,43 @@ def getResults(token, cookie: str, resl: str):
     :return [[Name, Note, Credits], ...]:
     """
     headers["Cookie"] = "cnsc=" + cookie
-    response = requests.request("GET", url + "?APPNAME=CampusNet&PRGNAME=COURSERESULTS&ARGUMENTS=-N" + token +
+    async with httpx.AsyncClient() as s:
-                                ",-N000307," + ",-N" + resl, headers=headers, data={})
+        response = await s.get(
-    html = BeautifulSoup(response.content.decode("utf-8"), 'lxml')
+            url=f"{url}?APPNAME=CampusNet&PRGNAME=COURSERESULTS&ARGUMENTS=-N{token},-N000307,,-N{resl}",
-    table = html.find('table', attrs={"class": "nb list"})
+            headers=headers)
-    body = table.find("tbody")
+        html = BeautifulSoup(response.content.decode("utf-8"), 'lxml')
-    vorl = body.find_all("tr")
+        table = html.find('table', attrs={"class": "nb list"})
-    vorlist = []
+        body = table.find("tbody")
-    for row in vorl:
+        vorl = body.find_all("tr")
-        cols = row.find_all("td")
+        vorlist = []
-        col = [[e.text.strip()] for e in cols]
+        tasks = []
-        if len(col) != 0:
+        i = 0
-            if len(col[4][0]) == 0:
+        for row in vorl:
-                col[2] = getPruefung(row.find("a")["href"])
+            cols = row.find_all("td")
+            col = [[e.text.strip()] for e in cols]
+            if len(col) != 0:
+                if len(col[4][0]) == 0:
+                    tasks += [getPruefung(s, row.find("a")["href"])]
+                    col[2] = i
+                    i += 1
             vorlist += [col[1:4]]
-    return vorlist
+        notlisted = await asyncio.gather(*tasks, return_exceptions=True)
+        for i in vorlist:
+            for e in range(0, len(i)):
+                if isinstance(i[e], int):
+                    i[e] = notlisted[i[e]]
+        return vorlist[:-1]
 
 
-def getPruefung(url):
+async def getPruefung(s, url):
     """
     Ermittelt Noten "geschachtelter" Prüfungen, die nicht auf der Hauptseite angezeigt werden.
     TODO: Namen der spezifischen Prüfungen auch zurückgeben, um Zusammensetzung zu spezifizieren.
+    :param s:
     :param url:
     :return list:
     """
-    response = requests.request("GET", "https://dualis.dhbw.de" + url, headers=headers, data={})
+    response = await s.get("https://dualis.dhbw.de" + url, headers=headers)
     html = BeautifulSoup(response.content.decode("utf-8"), 'lxml')
     table = html.find('table')
     pruefung = table.find_all("tr")

init.py

@@ -54,7 +54,17 @@ class Dualis(db.Model):
     token = db.Column(db.String(255), unique=True)
     uid = db.Column(db.Integer, primary_key=True)
     token_created = db.Column(db.Integer)
-    result_list = db.Column(db.String(15))
+    semester = db.Column(db.String(15))
+
+
+class Semesterlist(db.Model):
+    """
+    Datenbank-Modell für Semester-Liste.
+    """
+    uid = db.Column(db.Integer)
+    semestername = db.Column(db.String(25))
+    semesterid = db.Column(db.String(15))
+    itemid = db.Column(db.Integer, primary_key=True)
 
 
 class Meals(db.Model):

requesthelpers.py

@@ -1,3 +1,6 @@
+from init import Semesterlist
+
+
 def getCookie(cookies):
     """
     Liefert (letzten) Cookie der Cookies-Liste zurück.
@@ -8,3 +11,12 @@ def getCookie(cookies):
     for c in cookies:
         cookie = c.value
     return cookie
+
+
+def semesterlist(uid):
+    semesterlist = Semesterlist.query.filter_by(uid=uid).all()
+    semester = []
+    for s in semesterlist:
+        semester += [[s.semestername, s.semesterid]]
+    semester.sort(key=lambda x: x[-1], reverse=True)
+    return semester

routing.py

@@ -51,24 +51,26 @@ def welcome():
 
 @app.route("/theorie/noten", methods=["GET", "POST"])
 @login_required
-def displayNoten():
+async def displayNoten():
     """
     Zeigt die Noten aus Dualis an. Hierfür ist ein aktives Token nötig.
     :return HTML:
     """
     d = Dualis.query.filter_by(uid=current_user.id).first()
     if request.method == "POST":
-        d.result_list = request.form.get("sem")
+        d.semester = request.form.get("sem")
         db.session.commit()
-    if not d.result_list:
+    if not d.semester:
         return redirect(url_for("getSemester", next=url_for(request.endpoint)))
     t = d.token
-    sem = d.result_list
+    chosensemester = d.semester
     c = request.cookies.get("cnsc")
     timeout = fetchDUALIS.timeOut(d, c, "displayNoten")
     if timeout:
         return timeout
-    return render_template("noten.html", noten=fetchDUALIS.getResults(t, c, sem), semester=fetchDUALIS.getSem(t, c), sel=sem, s="n", praxis="hidden")
+    semester = requesthelpers.semesterlist(current_user.id)
+    noten = await fetchDUALIS.getResults(t, c, chosensemester)
+    return render_template("noten.html", noten=noten, semester=semester, sel=chosensemester, s="n", praxis="hidden")
 
 
 @app.route("/plan", methods=["GET"])
@@ -137,7 +139,7 @@ def redKurs():
 
 @app.route("/set-up/kurs")
 @login_required
-def getKurs():
+async def getKurs():
     """
     Automatische Kurs-Auswahl. \n
     Aktives Dualis-Token benötigt.
@@ -151,7 +153,7 @@ def getKurs():
             return timeout
         e = False
         if not current_user.kurs:
-            kurs = fetchDUALIS.getKurs(d.token, cookie)
+            kurs = await fetchDUALIS.getKurs(d.token, cookie)
             if kurs != 0:
                 if not fetchRAPLA.getIcal(kurs):
                     return render_template('kurs.html', detected=(kurs, e), s="s", theorie="hidden", praxis="hidden",
@@ -172,15 +174,23 @@ def getKurs():
 
 @app.route("/set-up/semester")
 @login_required
-def getSemester():
+async def getSemester():
     """
     Manuelle Semester-Auswahl.
     :return HTML:
     """
     t = Dualis.query.filter_by(uid=current_user.id).first().token
     c = request.cookies.get("cnsc")
-
+    semesterlist = Semesterlist.query.filter_by(uid=current_user.id).all()
-    return render_template("semester.html", semester=fetchDUALIS.getSem(t, c), s="s", theorie="hidden", praxis="hidden")
+    if not semesterlist:
+        semester = await fetchDUALIS.getSem(t, c)
+        for i in semester:
+            semitem = Semesterlist(semestername=i[0], semesterid=i[1], uid=current_user.id, itemid=current_user.id*int(i[1][-7:])//1000000)
+            db.session.add(semitem)
+        db.session.commit()
+    else:
+        semester = requesthelpers.semesterlist(current_user.id)
+    return render_template("semester.html", semester=semester, s="s", theorie="hidden", praxis="hidden")
 
 
 @app.route("/set-up/semester", methods=["POST"])
@@ -194,7 +204,7 @@ def setSemester():
     if not n:
         n = url_for("welcome")
     d = Dualis.query.filter_by(uid=current_user.id).first()
-    d.result_list = request.form.get("sem")
+    d.semester = request.form.get("sem")
     db.session.commit()
     return redirect(n)
 
@@ -271,7 +281,7 @@ async def login_post():
         db.session.commit()
         login_user(user)
         if user.kurs:
-            if not dualis.result_list:
+            if not dualis.semester:
                 success = make_response(redirect(url_for("getSemester")))
             elif not n:
                 success = make_response(redirect(url_for("welcome")))
@@ -297,14 +307,14 @@ async def login_post():
 
 @app.route("/log-out")
 @login_required
-def logout():
+async def logout():
     """
     Loggt den User aus.
     :return Empty Token:
     """
     cookie = request.cookies.get("cnsc")
     dualis = Dualis.query.filter_by(uid=current_user.id).first()
-    fetchDUALIS.logOut(dualis.token, cookie)
+    await fetchDUALIS.logOut(dualis.token, cookie)
     dualis.token = None
     db.session.commit()
     logout_user()

tests_examples/vergleich.py

@@ -39,6 +39,7 @@ def celery_requests(self):
         return response, s
 
 
+# noinspection DuplicatedCode
 async def checkUser_celery():
     req = celery_requests.apply_async()
     response = req[0]
@@ -55,6 +56,7 @@ async def checkUser_celery():
     return token, cookie
 
 
+# noinspection DuplicatedCode
 def checkUser_normal():
     """
     Erhält von Dualis den Token und Cookie für User.
@@ -81,6 +83,7 @@ def checkUser_normal():
     return token, cookie
 
 
+# noinspection DuplicatedCode
 async def checkUser_async():
     """
     Erhält von Dualis den Token und Cookie für User.
@@ -88,6 +91,7 @@ async def checkUser_async():
     :param password:
     :return (Token, Session):
     """
+    # noinspection DuplicatedCode
     async with httpx.AsyncClient() as s:
         content = (f'usrname={fmail}&pass={fpw}&ARGUMENTS=clino%2Cusrname%2Cpass%2Cmenuno%2Cmenu_type%2Cbrowser'
                    f'%2Cplatform&APPNAME=CampusNet&PRGNAME=LOGINCHECK')
@@ -104,6 +108,7 @@ async def checkUser_async():
         return token, cookie
 
 
+# noinspection DuplicatedCode
 def getSem_normal(token: int, cookie: str):
     """
     Liefert die Liste aller auf Dualis verfügbaren Semester.
@@ -127,6 +132,7 @@ def getSem_normal(token: int, cookie: str):
     return optlist
 
 
+# noinspection DuplicatedCode
 async def getSem_async(token, cookie):
     """
         Liefert die Liste aller auf Dualis verfügbaren Semester.
@@ -155,6 +161,7 @@ async def getSem_celery(token, cookie):
     pass
 
 
+# noinspection DuplicatedCode
 async def getResults_async(token, cookie, resl):
     headers["Cookie"] = "cnsc=" + cookie
     async with httpx.AsyncClient() as s:
@@ -185,7 +192,7 @@ async def getResults_async(token, cookie, resl):
                     i[e] = extrakurse[i[e]]
         return vorlist[:-1]
 
-
+# noinspection DuplicatedCode
 async def getPruefung_async(s, url):
     response = await s.get("https://dualis.dhbw.de" + url, headers=headers)
     html = BeautifulSoup(response.content.decode("utf-8"), 'lxml')
@@ -252,14 +259,14 @@ async def tests():
         end = time.perf_counter()
         normaltimeloop = end - start
 
-        fetchDUALIS.logOut(n[0], n[1])
+        await fetchDUALIS.logOut(n[0], n[1])
 
         start = time.perf_counter()
         a = await async_normal()
         end = time.perf_counter()
         asynctimeloop = end - start
 
-        fetchDUALIS.logOut(a[0], a[1])
+        await fetchDUALIS.logOut(a[0], a[1])
         if a[2] == n[2]:
             normaltime += normaltimeloop
             asynctime += asynctimeloop