Should probably work now. IDK. Need SSH. Anyway, I should focus on the csv format, as we won't be creating an interface for the data.
This commit is contained in:
parent
602420a3fb
commit
2644c48d81
64
src/main.py
64
src/main.py
@ -3,41 +3,75 @@ from bs4 import BeautifulSoup
|
|||||||
from chardet import detect
|
from chardet import detect
|
||||||
import csv
|
import csv
|
||||||
from json import dumps
|
from json import dumps
|
||||||
# Dummy
|
|
||||||
ROOT = "http://192.168.2.20"
|
ROOT = "http://192.168.2.20"
|
||||||
|
|
||||||
def bundle_file(url:str):
|
# Turns a csv reader into a list of dictionaries(json)
|
||||||
|
def reader_to_json(reader:csv.DictReader) -> list[dict]:
|
||||||
|
arr = []
|
||||||
|
for row in reader:
|
||||||
|
# No whitespaces
|
||||||
|
row = {k.strip():v.strip() for k,v in row.items()}
|
||||||
|
arr.append(row)
|
||||||
|
return arr
|
||||||
|
|
||||||
|
# Gets a csv file from the server and turns it into a list of dictionaries(json)
|
||||||
|
def get_json(url:str) -> list[dict]:
|
||||||
# get the file
|
# get the file
|
||||||
raw = get(url).content
|
raw = get(url).content
|
||||||
|
print(raw)
|
||||||
|
|
||||||
|
# Figure out the encoding
|
||||||
encoding = detect(raw)['encoding']
|
encoding = detect(raw)['encoding']
|
||||||
raw = raw.decode(encoding)
|
raw = raw.decode('utf-8')
|
||||||
|
|
||||||
# turn it into json
|
# turn it into json
|
||||||
reader = csv.DictReader(raw)
|
reader = csv.DictReader(raw)
|
||||||
for row in reader:
|
arr = reader_to_json(reader)
|
||||||
for key in row:
|
|
||||||
row[key] = row[key].strip()
|
|
||||||
arr.append(row)
|
|
||||||
|
|
||||||
# dump to file
|
|
||||||
with open("out.json", "w") as f:
|
|
||||||
f.write(dumps(arr, indent=2))
|
|
||||||
|
|
||||||
return arr
|
return arr
|
||||||
|
|
||||||
|
# DEBUG function. Loads a local file
|
||||||
|
def local_file(path:str) -> list[dict]:
|
||||||
|
with open(path, "r") as i:
|
||||||
|
reader:list = reader_to_json(csv.DictReader(i))
|
||||||
|
|
||||||
|
with open("out.json", "w") as o:
|
||||||
|
o.write(dumps(reader, indent=2))
|
||||||
|
|
||||||
def get_file(entries:int):
|
return reader
|
||||||
|
|
||||||
|
# Gets file from the server
|
||||||
|
def get_file(entries:int) -> dict:
|
||||||
url = f"{ROOT}/Portal/Portal.mwsl?PriNav=DataLog&RecentCount={entries}"
|
url = f"{ROOT}/Portal/Portal.mwsl?PriNav=DataLog&RecentCount={entries}"
|
||||||
soup = BeautifulSoup(get(url).content, "html.parser")
|
soup = BeautifulSoup(get(url).content, "html.parser")
|
||||||
link = soup.find(id="link1")
|
link = soup.find(id="link1")
|
||||||
|
|
||||||
return bundle_file(ROOT + link['href'])
|
if link:
|
||||||
|
link = link.get("href")
|
||||||
|
link = f"{ROOT}{link}"
|
||||||
|
return get_json(link)
|
||||||
|
|
||||||
|
raise ValueError("No link found")
|
||||||
|
|
||||||
|
# DEBUG: Get file locally
|
||||||
|
# return local_file("test.csv")
|
||||||
|
|
||||||
|
# Saves a json file
|
||||||
|
def save_file(json:list[dict], filename:str) -> None:
|
||||||
|
try:
|
||||||
|
with open(filename, "w") as o:
|
||||||
|
o.write(dumps(json, indent=2))
|
||||||
|
except Exception as exc:
|
||||||
|
raise IndexError("Error saving file") from exc
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
file = get_file(10)
|
# file = get_file(10)
|
||||||
print(file)
|
|
||||||
|
# DEBUG: local test file
|
||||||
|
file = local_file("test.csv")
|
||||||
|
# Save file
|
||||||
|
save_file(file, "out.json")
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
Loading…
x
Reference in New Issue
Block a user