120 lines
2.4 KiB
Python
120 lines
2.4 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import parseHomegate
|
|
import soup
|
|
|
|
|
|
from pprint import pprint
|
|
from push import push
|
|
from dbFlats import db
|
|
from pathlib import Path
|
|
|
|
ntfyLnk = 'https://push.kmuhrer.net'
|
|
ntfyTopic = 'Wohnungen'
|
|
|
|
dataPath = os.path.join(Path(os.path.dirname(__file__)).parent, 'data')
|
|
dbFile = dataPath + '/articlesWhg.db'
|
|
filterFile = dataPath + '/article_filterWhg.conf'
|
|
|
|
def jsonDump(data, file):
|
|
json.dump(data,open(file,'w'),indent=4)
|
|
|
|
|
|
def compaireArticles(new, old):
|
|
newArticles = []
|
|
for key in new:
|
|
if not key in old:
|
|
#found new
|
|
newArticles.append(new[key])
|
|
return newArticles
|
|
|
|
def readDBTable(table):
|
|
dbCon = db(dbFile)
|
|
try:
|
|
jd = json.dumps(dbCon.readAll(table))
|
|
except:
|
|
return 'none'
|
|
return jd
|
|
|
|
def listDBTable():
|
|
dbCon = db(dbFile)
|
|
try:
|
|
jd = json.dumps(dbCon.readTables())
|
|
except:
|
|
return 'none'
|
|
return jd
|
|
|
|
async def returnArticleContent(siId, href):
|
|
p = parseHomegate.parse('none')
|
|
|
|
s = soup.serve(href)
|
|
content = await s.htmlAsync()
|
|
|
|
try:
|
|
jd = json.dumps(await p.addArticleContent(content, siId, 'return'))
|
|
except:
|
|
return 'none'
|
|
return jd
|
|
|
|
|
|
async def run():
|
|
result = await main()
|
|
articleObj = json.loads(result)
|
|
articlesArray = []
|
|
|
|
dbCon = db(dbFile)
|
|
ntfy = push(ntfyLnk, ntfyTopic)
|
|
|
|
|
|
for line in dbCon.readAll('new_articles'):
|
|
articlesArray.append(line[1])
|
|
|
|
if len(articlesArray) <= 0:
|
|
for index in articleObj:
|
|
article = articleObj[index]
|
|
dbCon.writeNew(article)
|
|
exit
|
|
|
|
dbCon.delete_table('new_articles')
|
|
|
|
for index in articleObj:
|
|
article = articleObj[index]
|
|
dbCon.writeNew(article)
|
|
|
|
|
|
newArticles = compaireArticles(articleObj, articlesArray)
|
|
|
|
if not len(newArticles) <= 0:
|
|
for article in newArticles:
|
|
ntfy.send(article)
|
|
#pprint(article)
|
|
dbCon.write(article)
|
|
else:
|
|
ntfy.send('none')
|
|
pass
|
|
|
|
|
|
async def runAsync(func, ff: str):
|
|
return await asyncio.to_thread(func, ff)
|
|
|
|
|
|
async def main(mode = 'full'):
|
|
|
|
ff = filterFile
|
|
|
|
parsers = [parseHomegate.init(ff, mode)]
|
|
articles = {}
|
|
articlesHomegate = await asyncio.gather(*parsers)
|
|
|
|
articles = articlesHomegate[0]
|
|
|
|
#articles = {**articlesHomegate}
|
|
|
|
return json.dumps(articles)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
asyncio.run(run()) |