From a0fa33d5e114b0b4ccf1a1fdab4f2d216d41f12c Mon Sep 17 00:00:00 2001 From: mrbwburns <> Date: Mon, 20 Nov 2023 17:29:45 +0100 Subject: [PATCH] scraping the weather instead of using the API --- inkycal/modules/inkycal_openweather_scrape.py | 113 +++-- inkycal/modules/inkycal_weather.py | 460 +----------------- 2 files changed, 61 insertions(+), 512 deletions(-) diff --git a/inkycal/modules/inkycal_openweather_scrape.py b/inkycal/modules/inkycal_openweather_scrape.py index 1e652fa..db1b516 100644 --- a/inkycal/modules/inkycal_openweather_scrape.py +++ b/inkycal/modules/inkycal_openweather_scrape.py @@ -9,67 +9,72 @@ from selenium.webdriver.support.wait import WebDriverWait import time -# Set the desired viewport size (width, height) -my_width = 480 -my_height = 850 -mobile_emulation = { - "deviceMetrics": { "width": my_width, "height": my_height, "pixelRatio": 1.0 }, - "userAgent": "Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19" -} +def get_scraped_weatherforecast_image() -> Image: + # Set the desired viewport size (width, height) + my_width = 480 + my_height = 850 + mobile_emulation = { + "deviceMetrics": { "width": my_width, "height": my_height, "pixelRatio": 1.0 }, + "userAgent": "Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19" + } -# Create an instance of the webdriver with some pre-configured options -options = Options() -options.add_argument("--no-sandbox") -options.add_argument("--headless=new") -#options.add_argument("--disable-gpu") -options.add_argument("--disable-dev-shm-usage") -options.add_experimental_option("mobileEmulation", mobile_emulation) -service = Service("/usr/bin/chromedriver") -driver = webdriver.Chrome(service=service, options=options) + # Create an instance of the webdriver with some pre-configured options + options = Options() + options.add_argument("--no-sandbox") + options.add_argument("--headless=new") + #options.add_argument("--disable-gpu") + options.add_argument("--disable-dev-shm-usage") + options.add_experimental_option("mobileEmulation", mobile_emulation) + service = Service("/usr/bin/chromedriver") + driver = webdriver.Chrome(service=service, options=options) -# Navigate to webpage -driver.get("https://openweathermap.org/city/2867714") + # Navigate to webpage + driver.get("https://openweathermap.org/city/2867714") -# Wait and find the Cookie Button -login_button = driver.find_element(By.XPATH, '//*[@id="stick-footer-panel"]/div/div/div/div/div/button') -# Scroll to it -driver.execute_script("return arguments[0].scrollIntoView();", login_button) -# Click the button -button_clicked = False -while(button_clicked == False): - try: - login_button.click() - button_clicked = True - print("All cookies successfully accepted!") - except ElementClickInterceptedException: - print("Couldn't click the cookie button, retrying...") - time.sleep(10) + # Wait and find the Cookie Button + login_button = driver.find_element(By.XPATH, '//*[@id="stick-footer-panel"]/div/div/div/div/div/button') + # Scroll to it + driver.execute_script("return arguments[0].scrollIntoView();", login_button) + # Click the button + button_clicked = False + while(button_clicked == False): + try: + login_button.click() + button_clicked = True + print("All cookies successfully accepted!") + except ElementClickInterceptedException: + print("Couldn't click the cookie button, retrying...") + time.sleep(10) -# hacky wait statement for all the page elements to load -WebDriverWait(driver, timeout=20).until(lambda d: d.find_element(By.XPATH, '//*[@id="weather-widget"]/div[2]/div[1]/div[1]/div[1]/h2').text != "") + # hacky wait statement for all the page elements to load + WebDriverWait(driver, timeout=20).until(lambda d: d.find_element(By.XPATH, '//*[@id="weather-widget"]/div[2]/div[1]/div[1]/div[1]/h2').text != "") -# Scroll to the start of the forecast -forecast_element = driver.find_element(By.XPATH, '//*[@id="weather-widget"]/div[2]/div[1]/div[1]/div[1]') -driver.execute_script("return arguments[0].scrollIntoView();", forecast_element) + # Scroll to the start of the forecast + forecast_element = driver.find_element(By.XPATH, '//*[@id="weather-widget"]/div[2]/div[1]/div[1]/div[1]') + driver.execute_script("return arguments[0].scrollIntoView();", forecast_element) -# remove the map -map_element = driver.find_element(By.XPATH, '//*[@id="weather-widget"]/div[2]/div[1]/div[2]') -driver.execute_script("arguments[0].remove();", map_element) + # remove the map + map_element = driver.find_element(By.XPATH, '//*[@id="weather-widget"]/div[2]/div[1]/div[2]') + driver.execute_script("arguments[0].remove();", map_element) -# zoom in a little - not now -#driver.execute_script("document.body.style.zoom='110%'"); + # zoom in a little - not now + #driver.execute_script("document.body.style.zoom='110%'"); -# Save as a screenshot -image_filename = "openweather_scraped.png" -driver.save_screenshot(image_filename) + html_element = driver.find_element(By.TAG_NAME, "html") + driver.execute_script("arguments[0].style.fontSize = '16px';", html_element) -# Close the WebDriver when done -driver.quit() + # Save as a screenshot + image_filename = "openweather_scraped.png" + driver.save_screenshot(image_filename) -# crop, resize, enhance & rotate the image for inky display -im = Image.open(image_filename, mode='r', formats=None) -im = im.crop((0, 50, my_width, my_height)) -#im = im.resize((800, 480), Image.Resampling.LANCZOS) -#im = im.rotate(90, Image.NEAREST, expand = 1) -im = ImageEnhance.Contrast(im).enhance(1.5) -im.save(image_filename) + # Close the WebDriver when done + driver.quit() + + # crop, resize, enhance & rotate the image for inky display + im = Image.open(image_filename, mode='r', formats=None) + im = im.crop((0, 50, my_width, my_height)) + #im = im.resize((800, 480), Image.Resampling.LANCZOS) + #im = im.rotate(90, Image.NEAREST, expand = 1) + im = ImageEnhance.Contrast(im).enhance(1.3) + im.save(image_filename) + return im, im diff --git a/inkycal/modules/inkycal_weather.py b/inkycal/modules/inkycal_weather.py index 28d184a..2325f46 100644 --- a/inkycal/modules/inkycal_weather.py +++ b/inkycal/modules/inkycal_weather.py @@ -7,12 +7,10 @@ Copyright by aceinnolab from inkycal.modules.template import inkycal_module from inkycal.custom import * +from inkycal.modules.inkycal_openweather_scrape import get_scraped_weatherforecast_image import math import decimal -import arrow - -from pyowm.owm import OWM logger = logging.getLogger(__name__) @@ -107,462 +105,8 @@ class Weather(inkycal_module): def generate_image(self): """Generate image for this module""" - # Define new image size with respect to padding - im_width = int(self.width - (2 * self.padding_left)) - im_height = int(self.height - (2 * self.padding_top)) - im_size = im_width, im_height - logger.info(f'Image size: {im_size}') - - # Create an image for black pixels and one for coloured pixels - im_black = Image.new('RGB', size=im_size, color='white') - im_colour = Image.new('RGB', size=im_size, color='white') - - # Check if internet is available - if internet_available(): - logger.info('Connection test passed') - else: - raise NetworkNotReachableError - - def get_moon_phase(): - """Calculate the current (approximate) moon phase""" - - dec = decimal.Decimal - diff = now - arrow.get(2001, 1, 1) - days = dec(diff.days) + (dec(diff.seconds) / dec(86400)) - lunations = dec("0.20439731") + (days * dec("0.03386319269")) - position = lunations % dec(1) - index = math.floor((position * dec(8)) + dec("0.5")) - return { - 0: '\uf095', - 1: '\uf099', - 2: '\uf09c', - 3: '\uf0a0', - 4: '\uf0a3', - 5: '\uf0a7', - 6: '\uf0aa', - 7: '\uf0ae'}[int(index) & 7] - - def is_negative(temp): - """Check if temp is below freezing point of water (0°C/30°F) - returns True if temp below freezing point, else False""" - answer = False - - if temp_unit == 'celsius' and round(float(temp.split('°')[0])) <= 0: - answer = True - elif temp_unit == 'fahrenheit' and round(float(temp.split('°')[0])) <= 0: - answer = True - return answer - - # Lookup-table for weather icons and weather codes - weathericons = { - '01d': '\uf00d', - '02d': '\uf002', - '03d': '\uf013', - '04d': '\uf012', - '09d': '\uf01a', - '10d': '\uf019', - '11d': '\uf01e', - '13d': '\uf01b', - '50d': '\uf014', - '01n': '\uf02e', - '02n': '\uf013', - '03n': '\uf013', - '04n': '\uf013', - '09n': '\uf037', - '10n': '\uf036', - '11n': '\uf03b', - '13n': '\uf038', - '50n': '\uf023' - } - - def draw_icon(image, xy, box_size, icon, rotation=None): - """Custom function to add icons of weather font on image - image = on which image should the text be added? - xy = xy-coordinates as tuple -> (x,y) - box_size = size of text-box -> (width,height) - icon = icon-unicode, looks this up in weathericons dictionary - """ - - icon_size_correction = { - '\uf00d': 10 / 60, - '\uf02e': 51 / 150, - '\uf019': 21 / 60, - '\uf01b': 21 / 60, - '\uf0b5': 51 / 150, - '\uf050': 25 / 60, - '\uf013': 51 / 150, - '\uf002': 0, - '\uf031': 29 / 100, - '\uf015': 21 / 60, - '\uf01e': 52 / 150, - '\uf056': 51 / 150, - '\uf053': 14 / 150, - '\uf012': 51 / 150, - '\uf01a': 51 / 150, - '\uf014': 51 / 150, - '\uf037': 42 / 150, - '\uf036': 42 / 150, - '\uf03b': 42 / 150, - '\uf038': 42 / 150, - '\uf023': 35 / 150, - '\uf07a': 35 / 150, - '\uf051': 18 / 150, - '\uf052': 18 / 150, - '\uf0aa': 0, - '\uf095': 0, - '\uf099': 0, - '\uf09c': 0, - '\uf0a0': 0, - '\uf0a3': 0, - '\uf0a7': 0, - '\uf0aa': 0, - '\uf0ae': 0 - } - - x, y = xy - box_width, box_height = box_size - text = icon - font = self.weatherfont - - # Increase fontsize to fit specified height and width of text box - size = 8 - font = ImageFont.truetype(font.path, size) - text_width, text_height = font.getsize(text) - - while (text_width < int(box_width * 0.9) and - text_height < int(box_height * 0.9)): - size += 1 - font = ImageFont.truetype(font.path, size) - text_width, text_height = font.getsize(text) - - text_width, text_height = font.getsize(text) - - # Align text to desired position - x = int((box_width / 2) - (text_width / 2)) - y = int((box_height / 2) - (text_height / 2) - (icon_size_correction[icon] * size) / 2) - - # Draw the text in the text-box - draw = ImageDraw.Draw(image) - print(f"Box width: {box_width}, Box height: {box_height}") - # TODO: fix ValueError: Width and height must be >= 0 - space = Image.new('RGBA', (box_width, box_height)) - ImageDraw.Draw(space).text((x, y), text, fill='black', font=font) - - if rotation != None: - space.rotate(rotation, expand=True) - - # Update only region with text (add text with transparent background) - image.paste(space, xy, space) - - # column1 column2 column3 column4 column5 column6 column7 - # |----------|----------|----------|----------|----------|----------|----------| - # | time | temperat.| moonphase| forecast1| forecast2| forecast3| forecast4| - # | current |----------|----------|----------|----------|----------|----------| - # | weather | humidity | sunrise | icon1 | icon2 | icon3 | icon4 | - # | icon |----------|----------|----------|----------|----------|----------| - # | | windspeed| sunset | temperat.| temperat.| temperat.| temperat.| - # |----------|----------|----------|----------|----------|----------|----------| - - # Calculate size rows and columns - col_width = im_width // 7 - - # Ratio width height - image_ratio = im_width / im_height - - if image_ratio >= 4: - row_height = im_height // 3 - else: - logger.info('Please consider decreasing the height.') - row_height = int((im_height * (1 - im_height / im_width)) / 3) - - _, text_height = self.font.getsize("Text") - row_height = max(row_height, text_height) - logger.debug(f"row_height: {row_height} | col_width: {col_width}") - - # Calculate spacings for better centering - spacing_top = int((im_width % col_width) / 2) - spacing_left = int((im_height % row_height) / 2) - - # Define sizes for weather icons - icon_small = int(col_width / 3) - icon_medium = icon_small * 2 - icon_large = icon_small * 3 - - # Calculate the x-axis position of each col - col1 = spacing_top - col2 = col1 + col_width - col3 = col2 + col_width - col4 = col3 + col_width - col5 = col4 + col_width - col6 = col5 + col_width - col7 = col6 + col_width - - # Calculate the y-axis position of each row - line_gap = int((im_height - spacing_top - 3 * row_height) // 4) - - row1 = line_gap - row2 = row1 + line_gap + row_height - row3 = row2 + line_gap + row_height - - # Draw lines on each row and border - ############################################################################ - ## draw = ImageDraw.Draw(im_black) - ## draw.line((0, 0, im_width, 0), fill='red') - ## draw.line((0, im_height-1, im_width, im_height-1), fill='red') - ## draw.line((0, row1, im_width, row1), fill='black') - ## draw.line((0, row1+row_height, im_width, row1+row_height), fill='black') - ## draw.line((0, row2, im_width, row2), fill='black') - ## draw.line((0, row2+row_height, im_width, row2+row_height), fill='black') - ## draw.line((0, row3, im_width, row3), fill='black') - ## draw.line((0, row3+row_height, im_width, row3+row_height), fill='black') - ############################################################################ - - # Positions for current weather details - weather_icon_pos = (col1, 0) - temperature_icon_pos = (col2, row1) - temperature_pos = (col2 + icon_small, row1) - humidity_icon_pos = (col2, row2) - humidity_pos = (col2 + icon_small, row2) - windspeed_icon_pos = (col2, row3) - windspeed_pos = (col2 + icon_small, row3) - - # Positions for sunrise, sunset, moonphase - moonphase_pos = (col3, row1) - sunrise_icon_pos = (col3, row2) - sunrise_time_pos = (col3 + icon_small, row2) - sunset_icon_pos = (col3, row3) - sunset_time_pos = (col3 + icon_small, row3) - - # Positions for forecast 1 - stamp_fc1 = (col4, row1) - icon_fc1 = (col4, row1 + row_height) - temp_fc1 = (col4, row3) - - # Positions for forecast 2 - stamp_fc2 = (col5, row1) - icon_fc2 = (col5, row1 + row_height) - temp_fc2 = (col5, row3) - - # Positions for forecast 3 - stamp_fc3 = (col6, row1) - icon_fc3 = (col6, row1 + row_height) - temp_fc3 = (col6, row3) - - # Positions for forecast 4 - stamp_fc4 = (col7, row1) - icon_fc4 = (col7, row1 + row_height) - temp_fc4 = (col7, row3) - - # Create current-weather and weather-forecast objects - if self.location.isdigit(): - logging.debug('looking up location by ID') - weather = self.owm.weather_at_id(int(self.location)).weather - forecast = self.owm.forecast_at_id(int(self.location), '3h') - else: - logging.debug('looking up location by string') - weather = self.owm.weather_at_place(self.location).weather - forecast = self.owm.forecast_at_place(self.location, '3h') - - # Set decimals - dec_temp = None if self.round_temperature == True else 1 - dec_wind = None if self.round_windspeed == True else 1 - - # Set correct temperature units - if self.units == 'metric': - temp_unit = 'celsius' - elif self.units == 'imperial': - temp_unit = 'fahrenheit' - - logging.debug(f'temperature unit: {temp_unit}') - logging.debug(f'decimals temperature: {dec_temp} | decimals wind: {dec_wind}') - - # Get current time - now = arrow.utcnow() - now = now.to("local") - - if self.forecast_interval == 'hourly': - - logger.debug("getting hourly forecasts") - - # Forecasts are provided for every 3rd full hour - # find out how many hours there are until the next 3rd full hour - if (now.hour % 3) != 0: - hour_gap = 3 - (now.hour % 3) - else: - hour_gap = 3 - - # Create timings for hourly forcasts - forecast_timings = [now.shift(hours=+ hour_gap + _).floor('hour') - for _ in range(0, 12, 3)] - - # Create forecast objects for given timings - forecasts = [forecast.get_weather_at(forecast_time.datetime) for - forecast_time in forecast_timings] - - # Add forecast-data to fc_data dictionary - fc_data = {} - for forecast in forecasts: - temp = '{}°'.format(round( - forecast.temperature(unit=temp_unit)['temp'], ndigits=dec_temp)) - - icon = forecast.weather_icon_name - fc_data['fc' + str(forecasts.index(forecast) + 1)] = { - 'temp': temp, - 'icon': icon, - 'stamp': forecast_timings[forecasts.index(forecast)].to( - get_system_tz()).format('H.00' if self.hour_format == 24 else 'h a') - } - - elif self.forecast_interval == 'daily': - - logger.debug("getting daily forecasts") - - def calculate_forecast(days_from_today): - """Get temperature range and most frequent icon code for forecast - days_from_today should be int from 1-4: e.g. 2 -> 2 days from today - """ - - # Create a list containing time-objects for every 3rd hour of the day - time_range = list(arrow.Arrow.range('hour', - now.shift(days=days_from_today).floor('day'), - now.shift(days=days_from_today).ceil('day') - ))[::3] - - # Get forecasts for each time-object - forecasts = [forecast.get_weather_at(_.datetime) for _ in time_range] - - # Get all temperatures for this day - daily_temp = [round(_.temperature(unit=temp_unit)['temp'], - ndigits=dec_temp) for _ in forecasts] - # Calculate min. and max. temp for this day - temp_range = f'{max(daily_temp)}°/{min(daily_temp)}°' - - # Get all weather icon codes for this day - daily_icons = [_.weather_icon_name for _ in forecasts] - # Find most common element from all weather icon codes - status = max(set(daily_icons), key=daily_icons.count) - - weekday = now.shift(days=days_from_today).format('ddd', locale= - self.locale) - return {'temp': temp_range, 'icon': status, 'stamp': weekday} - - forecasts = [calculate_forecast(days) for days in range(1, 5)] - - fc_data = {} - for forecast in forecasts: - fc_data['fc' + str(forecasts.index(forecast) + 1)] = { - 'temp': forecast['temp'], - 'icon': forecast['icon'], - 'stamp': forecast['stamp'] - } - - for key, val in fc_data.items(): - logger.debug((key, val)) - - # Get some current weather details - temperature = '{}°'.format(round( - weather.temperature(unit=temp_unit)['temp'], ndigits=dec_temp)) - - weather_icon = weather.weather_icon_name - humidity = str(weather.humidity) - sunrise_raw = arrow.get(weather.sunrise_time()).to(self.timezone) - sunset_raw = arrow.get(weather.sunset_time()).to(self.timezone) - - logger.debug(f'weather_icon: {weather_icon}') - - if self.hour_format == 12: - logger.debug('using 12 hour format for sunrise/sunset') - sunrise = sunrise_raw.format('h:mm a') - sunset = sunset_raw.format('h:mm a') - - elif self.hour_format == 24: - logger.debug('using 24 hour format for sunrise/sunset') - sunrise = sunrise_raw.format('H:mm') - sunset = sunset_raw.format('H:mm') - - # Format the windspeed to user preference - if self.use_beaufort: - logger.debug("using beaufort for wind") - wind = str(weather.wind(unit='beaufort')['speed']) - - else: - - if self.units == 'metric': - logging.debug('getting windspeed in metric unit') - wind = str(weather.wind(unit='meters_sec')['speed']) + 'm/s' - - elif self.units == 'imperial': - logging.debug('getting windspeed in imperial unit') - wind = str(weather.wind(unit='miles_hour')['speed']) + 'miles/h' - - dec = decimal.Decimal - moonphase = get_moon_phase() - - # Fill weather details in col 1 (current weather icon) - draw_icon(im_colour, weather_icon_pos, (col_width, im_height), - weathericons[weather_icon]) - - # Fill weather details in col 2 (temp, humidity, wind) - draw_icon(im_colour, temperature_icon_pos, (icon_small, row_height), - '\uf053') - - if is_negative(temperature): - write(im_black, temperature_pos, (col_width - icon_small, row_height), - temperature, font=self.font) - else: - write(im_black, temperature_pos, (col_width - icon_small, row_height), - temperature, font=self.font) - - draw_icon(im_colour, humidity_icon_pos, (icon_small, row_height), - '\uf07a') - - write(im_black, humidity_pos, (col_width - icon_small, row_height), - humidity + '%', font=self.font) - - draw_icon(im_colour, windspeed_icon_pos, (icon_small, icon_small), - '\uf050') - - write(im_black, windspeed_pos, (col_width - icon_small, row_height), - wind, font=self.font) - - # Fill weather details in col 3 (moonphase, sunrise, sunset) - draw_icon(im_colour, moonphase_pos, (col_width, row_height), moonphase) - - draw_icon(im_colour, sunrise_icon_pos, (icon_small, icon_small), '\uf051') - write(im_black, sunrise_time_pos, (col_width - icon_small, row_height), - sunrise, font=self.font) - - draw_icon(im_colour, sunset_icon_pos, (icon_small, icon_small), '\uf052') - write(im_black, sunset_time_pos, (col_width - icon_small, row_height), sunset, - font=self.font) - - # Add the forecast data to the correct places - for pos in range(1, len(fc_data) + 1): - stamp = fc_data[f'fc{pos}']['stamp'] - - icon = weathericons[fc_data[f'fc{pos}']['icon']] - temp = fc_data[f'fc{pos}']['temp'] - - write(im_black, eval(f'stamp_fc{pos}'), (col_width, row_height), - stamp, font=self.font) - draw_icon(im_colour, eval(f'icon_fc{pos}'), (col_width, row_height + line_gap * 2), - icon) - write(im_black, eval(f'temp_fc{pos}'), (col_width, row_height), - temp, font=self.font) - - border_h = row3 + row_height - border_w = col_width - 3 # leave 3 pixels gap - - # Add borders around each sub-section - draw_border(im_black, (col1, row1), (col_width * 3 - 3, border_h), - shrinkage=(0, 0)) - - for _ in range(4, 8): - draw_border(im_black, (eval(f'col{_}'), row1), (border_w, border_h), - shrinkage=(0, 0)) - # return the images ready for the display - return im_black, im_colour + return get_scraped_weatherforecast_image() if __name__ == '__main__':