165 lines
4.0 KiB
Python
165 lines
4.0 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import parseHomegate
|
|
import soup
|
|
|
|
|
|
from pprint import pprint
|
|
from push import push
|
|
from dbFlats import db
|
|
from pathlib import Path
|
|
|
|
ntfyLnk = 'https://push.kmuhrer.net'
|
|
ntfyTopic = 'Wohnungen'
|
|
maxPushMsg = 25
|
|
|
|
dataPath = os.path.join(Path(os.path.dirname(__file__)).parent, 'data')
|
|
dbFile = dataPath + '/articlesWhg.db'
|
|
filterFile = dataPath + '/article_filterWhg.conf'
|
|
|
|
def jsonDump(data, file):
|
|
json.dump(data,open(file,'w'),indent=4)
|
|
|
|
def prioSort(articles):
|
|
sortArray1 = []
|
|
sortArray2 = []
|
|
sortArray3 = []
|
|
for article in articles:
|
|
if article['prio'] == 'pinned':
|
|
sortArray1.append(article)
|
|
elif article['prio'] == 'important':
|
|
sortArray2.append(article)
|
|
else:
|
|
sortArray3.append(article)
|
|
|
|
#return sortArray1 + sortArray2 + sortArray3
|
|
sortArray = sortArray3 + sortArray2
|
|
return sortArray + sortArray1
|
|
|
|
def compaireArticles(new, old):
|
|
newArticles = []
|
|
for key in new:
|
|
if not key in old:
|
|
#found new
|
|
newArticles.append(new[key])
|
|
return newArticles
|
|
|
|
def readDBTable(table):
|
|
dbCon = db(dbFile)
|
|
try:
|
|
jd = json.dumps(dbCon.readAll(table))
|
|
except:
|
|
return 'none'
|
|
return jd
|
|
|
|
def listDBTable():
|
|
dbCon = db(dbFile)
|
|
try:
|
|
jd = json.dumps(dbCon.readTables())
|
|
except:
|
|
return 'none'
|
|
return jd
|
|
|
|
async def returnArticleContent(siId, href):
|
|
p = parseHomegate.parse('none')
|
|
|
|
s = soup.serve(href)
|
|
content = await s.htmlAsync()
|
|
|
|
try:
|
|
jd = json.dumps(await p.addArticleContent(content, siId, 'return'))
|
|
except:
|
|
return 'none'
|
|
return jd
|
|
|
|
|
|
|
|
async def run():
|
|
result = await main('quick')
|
|
currentArticles = json.loads(result)
|
|
dbNewArticles = []
|
|
|
|
dbCon = db(dbFile)
|
|
ntfy = push(ntfyLnk, ntfyTopic)
|
|
|
|
|
|
for line in dbCon.readAll('new_articles'):
|
|
dbNewArticles.append(line[1])
|
|
|
|
#check if an error occured
|
|
#check if new articles table exists if not create and exit
|
|
#check if there are new articles and replace new articles table
|
|
errorCheck = True
|
|
if 'INFO-OBJECT' in currentArticles.keys():
|
|
if currentArticles['INFO-OBJECT']['Titel'] == 'ERROR':
|
|
errorCheck = False
|
|
if errorCheck:
|
|
if len(dbNewArticles) <= 0:
|
|
for index in currentArticles:
|
|
if index == 'INFO-OBJECT': continue
|
|
article = currentArticles[index]
|
|
dbCon.writeNew(article)
|
|
exit
|
|
|
|
if not len(currentArticles) <= 0:
|
|
dbCon.delete_table('new_articles')
|
|
for index in currentArticles:
|
|
if index == 'INFO-OBJECT': continue
|
|
article = currentArticles[index]
|
|
dbCon.writeNew(article)
|
|
#compaire db and current articles
|
|
newArticles = compaireArticles(currentArticles, dbNewArticles)
|
|
|
|
#if there are new articles push them and write permanently to db
|
|
if not len(newArticles) <= 0:
|
|
|
|
#sort new articles by priority
|
|
newArticles = prioSort(newArticles)
|
|
|
|
c = 0
|
|
toomuchflag = False
|
|
newArticlesLen = len(newArticles)
|
|
pushTreshhold = newArticlesLen-maxPushMsg
|
|
for article in newArticles:
|
|
|
|
if c > pushTreshhold:
|
|
ntfy.send(article)
|
|
else:
|
|
toomuchflag = True
|
|
|
|
if article['id'] == 'INFO-OBJECT': continue
|
|
dbCon.write(article)
|
|
c += 1
|
|
|
|
if toomuchflag:
|
|
ntfy.send(f'there are {pushTreshhold} not shown articles')
|
|
else:
|
|
ntfy.send('none')
|
|
|
|
|
|
|
|
|
|
async def runAsync(func, ff: str):
|
|
return await asyncio.to_thread(func, ff)
|
|
|
|
|
|
async def main(mode = 'full'):
|
|
|
|
ff = filterFile
|
|
|
|
parsers = [parseHomegate.init(ff, mode)]
|
|
articles = {}
|
|
articlesHomegate = await asyncio.gather(*parsers)
|
|
|
|
articles = articlesHomegate[0]
|
|
|
|
#articles = {**articlesHomegate}
|
|
|
|
return json.dumps(articles)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
asyncio.run(run()) |