- improve devcontainer setup
- add openweather web scraper module
This commit is contained in:
parent
a1155184c4
commit
7cac3fd195
10
.devcontainer/Dockerfile
Normal file
10
.devcontainer/Dockerfile
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
FROM python:3.9-slim-bullseye
|
||||||
|
WORKDIR /usr/src/app
|
||||||
|
RUN apt-get -y update && apt-get install -yqq dos2unix chromium chromium-driver \
|
||||||
|
libxi6 libgconf-2-4 python3-selenium \
|
||||||
|
tzdata git
|
||||||
|
RUN git config --global --add safe.directory /usr/src/app
|
||||||
|
RUN python3 -m pip install --upgrade pip
|
||||||
|
RUN python3 -m pip install --user virtualenv
|
||||||
|
ENV TZ=Europe/Berlin
|
||||||
|
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
@ -1,18 +1,22 @@
|
|||||||
// For format details, see https://aka.ms/devcontainer.json.
|
// For format details, see https://aka.ms/devcontainer.json.
|
||||||
{
|
{
|
||||||
"name": "Inkycal-dev",
|
"name": "Inkycal-dev",
|
||||||
"image": "python:3.9-bullseye",
|
"build": {
|
||||||
|
"dockerfile": "Dockerfile"
|
||||||
|
},
|
||||||
|
|
||||||
// This is the settings.json mount
|
// This is the settings.json mount
|
||||||
"mounts": ["source=/c/temp/settings_test.json,target=/boot/settings.json,type=bind,consistency=cached"],
|
"mounts": ["source=/c/temp/settings_test.json,target=/boot/settings.json,type=bind,consistency=cached"],
|
||||||
|
|
||||||
// Use 'postCreateCommand' to run commands after the container is created.
|
// Use 'postCreateCommand' to run commands after the container is created.
|
||||||
"postCreateCommand": "pip3 install --upgrade pip && pip3 install --user -r requirements.txt",
|
"postCreateCommand": "chmod +x ./.devcontainer/postCreate.sh && ./.devcontainer/postCreate.sh",
|
||||||
|
|
||||||
"customizations": {
|
"customizations": {
|
||||||
"vscode": {
|
"vscode": {
|
||||||
"extensions": [
|
"extensions": [
|
||||||
"ms-python.python"
|
"ms-python.python",
|
||||||
|
"ms-python.black-formatter",
|
||||||
|
"ms-azuretools.vscode-docker"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
2
.devcontainer/postCreate.sh
Normal file
2
.devcontainer/postCreate.sh
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
source ./venv/Scripts/activate && pip3 install --upgrade pip && pip3 install --user -r requirements.txt
|
75
inkycal/modules/inkycal_openweather_scrape.py
Normal file
75
inkycal/modules/inkycal_openweather_scrape.py
Normal file
@ -0,0 +1,75 @@
|
|||||||
|
from PIL import Image
|
||||||
|
from PIL import ImageEnhance
|
||||||
|
from selenium import webdriver
|
||||||
|
from selenium.common.exceptions import ElementClickInterceptedException
|
||||||
|
from selenium.webdriver.chrome.options import Options
|
||||||
|
from selenium.webdriver.chrome.service import Service
|
||||||
|
from selenium.webdriver.common.by import By
|
||||||
|
from selenium.webdriver.support.wait import WebDriverWait
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
|
# Set the desired viewport size (width, height)
|
||||||
|
my_width = 480
|
||||||
|
my_height = 850
|
||||||
|
mobile_emulation = {
|
||||||
|
"deviceMetrics": { "width": my_width, "height": my_height, "pixelRatio": 1.0 },
|
||||||
|
"userAgent": "Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Create an instance of the webdriver with some pre-configured options
|
||||||
|
options = Options()
|
||||||
|
options.add_argument("--no-sandbox")
|
||||||
|
options.add_argument("--headless=new")
|
||||||
|
#options.add_argument("--disable-gpu")
|
||||||
|
options.add_argument("--disable-dev-shm-usage")
|
||||||
|
options.add_experimental_option("mobileEmulation", mobile_emulation)
|
||||||
|
service = Service("/usr/bin/chromedriver")
|
||||||
|
driver = webdriver.Chrome(service=service, options=options)
|
||||||
|
|
||||||
|
# Navigate to webpage
|
||||||
|
driver.get("https://openweathermap.org/city/2867714")
|
||||||
|
|
||||||
|
# Wait and find the Cookie Button
|
||||||
|
login_button = driver.find_element(By.XPATH, '//*[@id="stick-footer-panel"]/div/div/div/div/div/button')
|
||||||
|
# Scroll to it
|
||||||
|
driver.execute_script("return arguments[0].scrollIntoView();", login_button)
|
||||||
|
# Click the button
|
||||||
|
button_clicked = False
|
||||||
|
while(button_clicked == False):
|
||||||
|
try:
|
||||||
|
login_button.click()
|
||||||
|
button_clicked = True
|
||||||
|
print("All cookies successfully accepted!")
|
||||||
|
except ElementClickInterceptedException:
|
||||||
|
print("Couldn't click the cookie button, retrying...")
|
||||||
|
time.sleep(10)
|
||||||
|
|
||||||
|
# hacky wait statement for all the page elements to load
|
||||||
|
WebDriverWait(driver, timeout=20).until(lambda d: d.find_element(By.XPATH, '//*[@id="weather-widget"]/div[2]/div[1]/div[1]/div[1]/h2').text != "")
|
||||||
|
|
||||||
|
# Scroll to the start of the forecast
|
||||||
|
forecast_element = driver.find_element(By.XPATH, '//*[@id="weather-widget"]/div[2]/div[1]/div[1]/div[1]')
|
||||||
|
driver.execute_script("return arguments[0].scrollIntoView();", forecast_element)
|
||||||
|
|
||||||
|
# remove the map
|
||||||
|
map_element = driver.find_element(By.XPATH, '//*[@id="weather-widget"]/div[2]/div[1]/div[2]')
|
||||||
|
driver.execute_script("arguments[0].remove();", map_element)
|
||||||
|
|
||||||
|
# zoom in a little - not now
|
||||||
|
#driver.execute_script("document.body.style.zoom='110%'");
|
||||||
|
|
||||||
|
# Save as a screenshot
|
||||||
|
image_filename = "openweather_scraped.png"
|
||||||
|
driver.save_screenshot(image_filename)
|
||||||
|
|
||||||
|
# Close the WebDriver when done
|
||||||
|
driver.quit()
|
||||||
|
|
||||||
|
# crop, resize, enhance & rotate the image for inky display
|
||||||
|
im = Image.open(image_filename, mode='r', formats=None)
|
||||||
|
im = im.crop((0, 50, my_width, my_height))
|
||||||
|
#im = im.resize((800, 480), Image.Resampling.LANCZOS)
|
||||||
|
#im = im.rotate(90, Image.NEAREST, expand = 1)
|
||||||
|
im = ImageEnhance.Contrast(im).enhance(1.5)
|
||||||
|
im.save(image_filename)
|
@ -1,30 +1,56 @@
|
|||||||
|
appdirs==1.4.4
|
||||||
arrow==1.2.3
|
arrow==1.2.3
|
||||||
|
attrs==22.2.0
|
||||||
|
beautifulsoup4==4.12.2
|
||||||
certifi==2023.7.22
|
certifi==2023.7.22
|
||||||
|
charset-normalizer==3.2.0
|
||||||
|
contourpy==1.1.1
|
||||||
cycler==0.11.0
|
cycler==0.11.0
|
||||||
|
distlib==0.3.7
|
||||||
|
exceptiongroup==1.1.3
|
||||||
feedparser==6.0.10
|
feedparser==6.0.10
|
||||||
|
filelock==3.12.4
|
||||||
fonttools==4.40.0
|
fonttools==4.40.0
|
||||||
|
frozendict==2.3.8
|
||||||
geojson==2.3.0
|
geojson==2.3.0
|
||||||
|
h11==0.14.0
|
||||||
|
html5lib==1.1
|
||||||
icalendar==5.0.7
|
icalendar==5.0.7
|
||||||
|
idna==3.4
|
||||||
|
importlib-resources==6.1.0
|
||||||
kiwisolver==1.4.4
|
kiwisolver==1.4.4
|
||||||
lxml==4.9.2
|
lxml==4.9.2
|
||||||
matplotlib==3.7.1
|
matplotlib==3.7.1
|
||||||
multitasking==0.0.11
|
multitasking==0.0.11
|
||||||
numpy==1.25.0
|
numpy==1.25.0
|
||||||
|
outcome==1.2.0
|
||||||
packaging==23.1
|
packaging==23.1
|
||||||
pandas==2.0.2
|
pandas==2.0.2
|
||||||
Pillow==9.5.0
|
Pillow==9.5.0
|
||||||
|
platformdirs==3.10.0
|
||||||
pyowm==3.3.0
|
pyowm==3.3.0
|
||||||
pyparsing==3.1.0
|
pyparsing==3.1.0
|
||||||
PySocks==1.7.1
|
PySocks==1.7.1
|
||||||
python-dateutil==2.8.2
|
python-dateutil==2.8.2
|
||||||
|
python-dotenv==1.0.0
|
||||||
pytz==2023.3
|
pytz==2023.3
|
||||||
recurring-ical-events==2.0.2
|
recurring-ical-events==2.0.2
|
||||||
requests==2.31.0
|
requests==2.31.0
|
||||||
|
selenium==4.10.0
|
||||||
sgmllib3k==1.0.0
|
sgmllib3k==1.0.0
|
||||||
six==1.16.0
|
six==1.16.0
|
||||||
|
sniffio==1.3.0
|
||||||
|
sortedcontainers==2.4.0
|
||||||
|
soupsieve==2.5
|
||||||
todoist-api-python==2.0.2
|
todoist-api-python==2.0.2
|
||||||
|
trio==0.22.2
|
||||||
|
trio-websocket==0.10.4
|
||||||
typing_extensions==4.6.3
|
typing_extensions==4.6.3
|
||||||
|
tzdata==2023.3
|
||||||
urllib3==2.0.3
|
urllib3==2.0.3
|
||||||
|
virtualenv==20.24.5
|
||||||
|
webencodings==0.5.1
|
||||||
|
wsproto==1.2.0
|
||||||
|
x-wr-timezone==0.0.5
|
||||||
yfinance==0.2.21
|
yfinance==0.2.21
|
||||||
python-dotenv==1.0.0
|
zipp==3.17.0
|
||||||
setuptools==68.0.0
|
|
||||||
|
Loading…
Reference in New Issue
Block a user