diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..518486c --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__ + +Downloads + diff --git a/.trash/README.md b/.trash/README.md new file mode 100644 index 0000000..bc00108 --- /dev/null +++ b/.trash/README.md @@ -0,0 +1,2 @@ +# yankee-fire-control-code + diff --git a/.trash/RestAPIClient (copy 1).py b/.trash/RestAPIClient (copy 1).py new file mode 100644 index 0000000..cf7efb4 --- /dev/null +++ b/.trash/RestAPIClient (copy 1).py @@ -0,0 +1,37 @@ +import requests + +## https://www.nylas.com/blog/use-python-requests-module-rest-apis/#how-to-use-python-requests + +def callGET(url): + + print("enter -> callGET()") + + print("param / url / ",url) + + params = dict( + origin='Chicago,IL', + destination='Los+Angeles,CA', + waypoints='Joplin,MO|Oklahoma+City,OK', + sensor='false', + url2=url + ) + + headers = {'Accept': 'application/json'} + + print("params", params) + + response = requests.post("https://osiris.yankee.valorantdigital.com/channel/monthly/games/success/exists", headers=headers, data=params, params=params) + + print("response2", response.json()) + +def callPOST(): + + print("enter -> callPOST()") + + response = requests.post('https://httpbin.org/post', data = {'key':'value'}) + + print("response", response.json()) + +callGET("https://embanet.online") + +#callPOST() diff --git a/.trash/jmsclient.py b/.trash/jmsclient.py new file mode 100644 index 0000000..5e38140 --- /dev/null +++ b/.trash/jmsclient.py @@ -0,0 +1,42 @@ +import stomp +import time +import veneno +import fetcher + +class MyListener(stomp.ConnectionListener): + + def on_message(headers, message): + + conn.ack(headers['message-id']) + + print("** new message! **") + + print(message) + + fetcher.fetch(message.body) + + + + def on_error(headers, message): + + print('received an error "%s"' % message) + + conn.ack(headers['message-id']) + +conn = stomp.Connection([('osiris.yankee.valorantdigital.com', 61613)]) + +conn.set_listener('', MyListener()) + +conn.connect('admin', 'password', wait=True) + +conn.subscribe('/queue/sully-gnome-urls', id=1, ack='client', headers={'activemq.prefetchSize': 10}) + +print("## notification > connected to JMS server") + +while True: + + pass + + time.sleep(1) + +conn.disconnect() diff --git a/.trash/veneno.py b/.trash/veneno.py new file mode 100644 index 0000000..9cddfac --- /dev/null +++ b/.trash/veneno.py @@ -0,0 +1,19 @@ +# veneno.py + +def fetch(x): + + print(f"launch -> data fetcher -> url -> {x}") + +def eat(x,y): + + """Consumes x and y (pretend eating!). + + Args: + x: The first amount to be consumed. + y: The second amount to be consumed. + + Returns: + None + """ + + print(f"Eating {x} and {y}...") diff --git a/DiskUtils.py b/DiskUtils.py new file mode 100644 index 0000000..8c7727d --- /dev/null +++ b/DiskUtils.py @@ -0,0 +1,69 @@ +import os +import time +import re +import fnmatch + +def dowork(): + + ################################################################################################ + + url = "https://sullygnome.com/channel/kaicenat/2023january/streams" + + print("checking disk for the stuff...") + + print("url", url) + + fileExists = check_download_exists_matching_url(url) + + print("file already exists / ",fileExists) + + ################################################################################################ + +def check_download_exists_matching_url(url): + + print ("hello?") + + match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower()) + + if not match: + + print(f"Invalid URL format. Could not extract channel name, year, or month.") + + return + + channel_name, year, month = match.groups() + + print("channel / ",channel_name) + + print("year / ",year) + + print("month / ",month) + + #file_pattern = f"{channel_name}*{month}*{year}*.csv" + + ## this is a problem. it works but if there 2 channels. + ## redbull and redbull2 this is throw false positives. ugh + + file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower() + + for filename in os.listdir("/home/yankee/Downloads/"): + + updated = filename.lower() + + print("file in downloads / ",updated) + + if fnmatch.fnmatch(updated, file_pattern): + + print(f"Found matching file: {updated}") + + return True + + else: + + print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'.") + + return False + +def shutDownRobot(): + + print("shutting down the robot now") diff --git a/RestAPIClient.py b/RestAPIClient.py new file mode 100644 index 0000000..cea54df --- /dev/null +++ b/RestAPIClient.py @@ -0,0 +1,146 @@ +import requests +import json +import socket + +## https://www.nylas.com/blog/use-python-requests-module-rest-apis/#how-to-use-python-requests +## https://www.abstractapi.com/guides/ip-geolocation/get-ip-address-python + +def ip(): + + try: + + public_ip = requests.get('https://api.ipify.org').text + + print(f"Public IP Address: {public_ip}") + + return public_ip + + except requests.RequestException as e: + + print(f"Error retrieving public IP address: {e}") + + return None + +def addDownloadSuccess(url): + + print("enter -> addDownloadSuccess()") + + print("param / url / ",url) + + data = {"url": url, "hostIP": ip(),"ip": ip(),} + + json_data = json.dumps(data) + + print("param / data / ",data) + + print("param / data / ",json_data) + + headers = {"Content-Type": "application/json"} + + + + response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/success/add", data=json_data, headers=headers) + + print(response.text) + + +def addDownloadFailure(url): + + print("enter -> addDownloadFailure()") + + print("param / url / ",url) + + data = {"url": url, "hostIP": ip(),"ip": ip(),} + + json_data = json.dumps(data) + + print("param / data / ",data) + + print("param / data / ",json_data) + + headers = {"Content-Type": "application/json"} + + + + response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/failure/add", data=json_data, headers=headers) + + print(response.text) + + + + + + + + + + + + + + + + #print("params", params) + + #response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/success/exists", headers=headers, data=params, params=params) + + #print("response2", response.json()) + + +def searchURLSuccesses(url): + + print("enter -> callGET()") + + print("param / url / ",url) + + params = dict( + origin='Chicago,IL', + destination='Los+Angeles,CA', + waypoints='Joplin,MO|Oklahoma+City,OK', + sensor='false', + url2=url + ) + + + #data = {"name": "Jane Smith", "email": "janesmith@example.com", "url": url } + + data = { "url": url } + + headers = {"Content-Type": "application/json"} + + json_data = json.dumps(data) + + response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/success/exists", data=json_data, headers=headers) + + print(response.text) + + return response.text + +def searchURLFailures(url): + + print("enter -> callGET()") + + print("param / url / ",url) + + params = dict( + origin='Chicago,IL', + destination='Los+Angeles,CA', + waypoints='Joplin,MO|Oklahoma+City,OK', + sensor='false', + url2=url + ) + + + #data = {"name": "Jane Smith", "email": "janesmith@example.com", "url": url } + + data = { "url": url } + + headers = {"Content-Type": "application/json"} + + json_data = json.dumps(data) + + response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/failure/exists", data=json_data, headers=headers) + + print(response.text) + + return response.text diff --git a/SullyGnomeRobot.py b/SullyGnomeRobot.py new file mode 100644 index 0000000..7dcc6a0 --- /dev/null +++ b/SullyGnomeRobot.py @@ -0,0 +1,120 @@ +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +import time +from selenium.webdriver.support.select import Select +import RestAPIClient +import DiskUtils + +## https://sullygnome.com/channel/kaicenat/365/games" + +def download(url): + + print(f"launch -> SullyGnomeRobot -> download() -> {url}") + + driver = webdriver.Firefox() + + driver.get(url) + + ########################################## + + print("stage / start / tblControl_length check") + + try: + + dataset_drop_down_element = WebDriverWait(driver, 5).until( + + EC.presence_of_element_located((By.NAME, "tblControl_length")) + + ) + + dataset_drop_down_element = Select(dataset_drop_down_element) + + dataset_drop_down_element.select_by_visible_text("100") + + except Exception as e: + + print(f"Element with class name 'tblControl_length' not found: {e}") + + RestAPIClient.addDownloadFailure(url) + + driver.quit() + + return False + + print("stage / complete / tblControl_length check") + + ########################################## + + print("stage / start / TableExportLinkButton check") + + try: + + WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.CLASS_NAME, "TableExportLinkButton")) + ) + + link = driver.find_element(By.CLASS_NAME, "TableExportLinkButton") + + link.click() + + except Exception as e: + + print(f"Element with class name 'TableExportLinkButton' not found: {e}") + + RestAPIClient.addDownloadFailure(url) + + driver.quit() + + return False + + print("stage / complete / TableExportLinkButton check") + + ########################################## + + print("file download started. now checking if exist before closing") + + while True: + + print("checking is file exists... [loop]") + + existsAlready = DiskUtils.check_download_exists_matching_url(url) + + print("file already exists / ",existsAlready) + + if existsAlready: + + print(f"The file matching the url '{url}' exists.") + + print("sleeping for 4 secs before shutting down robot") + + time.sleep(4) + + print("shutting down the robot") + + driver.quit() + + print("robot shut down. breaking. and dying.") + + break + + else: + + print(f"The file matching the url '{url}' DOES NOT exist.") + + print("sleeping for some time before checking again..."); + + time.sleep(1) + + print(" file is finished downloading.moving on.") + + ########################################## + + RestAPIClient.addDownloadSuccess(url) + + print("stage / complete / database updated w/ success") + + print(f"complete -> SullyGnomeRobot -> download() -> {url}") diff --git a/consumer.bash b/consumer.bash new file mode 100644 index 0000000..e1db9d1 --- /dev/null +++ b/consumer.bash @@ -0,0 +1,46 @@ +#!/bin/bash + +## + +set -e + +set -x + +## + +reset + +clear + +## + +codeDir=/home/yankee/Code + +rm -rf $codeDir + +mkdir $codeDir + +ln -s /yankee/code $codeDir + +ls -lha $codeDir + +## + +downloadsDir=/home/yankee/Downloads + +rm -rf /home/yankee/Downloads + +##mkdir $downloadsDir + +ln -s /yankee/downloads/ /home/yankee/Downloads + +ls -lha $downloadsDir + +## + +echo "##" +echo "## launch > yankee gnome > consumer.py" +echo "##" + +python3 consumer.py + diff --git a/consumer.py b/consumer.py new file mode 100644 index 0000000..857ca1d --- /dev/null +++ b/consumer.py @@ -0,0 +1,76 @@ +import stomp +import time +import SullyGnomeRobot +import RestAPIClient + +class MyListener(stomp.ConnectionListener): + + def on_message(headers, message): + + print("** new message! **") + + print("message / ", message) + + print("messagebody / ", message.body) + + print("checking the value does or doesnt exist in db") + + alreadyExistsSuccess = RestAPIClient.searchURLSuccesses(message.body) + + print("alreadyExistsSuccess / ", alreadyExistsSuccess) + + if alreadyExistsSuccess.lower().strip() == "true": + + print("this record exists (successes). returning. doing nothing") + + return + + print("didn't exist in success. check in errors now.") + + alreadyExistsFailure = RestAPIClient.searchURLFailures(message.body) + + print("alreadyExistsFailure / ", alreadyExistsFailure) + + if alreadyExistsFailure.lower().strip() == "true": + + print("this record exists (failures). returning. doing nothing") + + return + + print("didn't exist in success or errors. will process now") + + try: + + SullyGnomeRobot.download(message.body) + + except Exception as e: + + print(f"exception / something went wrong when the sully gnome robot was downloading: {e}") + + RestAPIClient.addDownloadFailure(message.body) + + def on_error(headers, message): + + print('received an error "%s"' % message) + + RestAPIClient.addDownloadFailure(message.body) + +conn = stomp.Connection([('osiris.yankee.embanet.online', 61613)]) + +conn.set_listener('', MyListener()) + +conn.connect('admin', 'password', wait=True) + +conn.subscribe('/queue/yankee-sully-channels-monthly', id=1, ack='auto', headers={'activemq.prefetchSize': 10}) + +print("## ") +print("## notification > connected to JMS server (yankee-sully-channels-monthly)") +print("## ") + +while True: + + pass + + time.sleep(1) + +conn.disconnect() diff --git a/downloadcheck/fastdownload.bash b/downloadcheck/fastdownload.bash new file mode 100644 index 0000000..7442c1d --- /dev/null +++ b/downloadcheck/fastdownload.bash @@ -0,0 +1,9 @@ +set -e + +set -x + +reset + +clear + +python3 fastdownload.py \ No newline at end of file diff --git a/downloadcheck/fastdownload.py b/downloadcheck/fastdownload.py new file mode 100644 index 0000000..efb2935 --- /dev/null +++ b/downloadcheck/fastdownload.py @@ -0,0 +1,89 @@ +import os +import time +import re + +def dowork(): + + ################################################################################################ + + url = "https://sullygnome.com/channel/kaicenat/2023january/streams" + + print("checking disk for the stuff...") + + print("url", url) + + fileExists = check_download_exists_matching_url(url) + + print("file already exists / ",fileExists) + + ################################################################################################ + +def check_download_exists_matching_url(url): + + print ("hello?") + + match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/streams", url.lower()) + + if not match: + + print(f"Invalid URL format. Could not extract channel name, year, or month.") + + return + + channel_name, year, month = match.groups() + + print("channel / ",channel_name) + + print("year / ",year) + + print("month / ",month) + + #file_pattern = f"{channel_name}*{month}*{year}*.csv" + + file_pattern = f"{channel_name} - Twitch stream stats in {month} {year} - SullyGnome.csv".lower() + + for filename in os.listdir("/home/softwareshinobi/Downloads/"): + + updated = filename.lower() + + print("updated / ",updated) + + if re.match(file_pattern, updated): + + print(f"Found matching file: {updated}") + + return True + + else: + + print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'.") + + return False + +def shutDownRobot(): + + print("shutting down the robot now") + +while True: + + print("checking is file exists... [loop]") + + if check_file(file_to_check): + + print(f"The file '{file_to_check}' exists.") + + print("shutting down the robot") + + shutDownRobot() + + print("robot shut down. breaking. and dying.") + + break + + else: + + print(f"The file '{file_to_check}' does not exist.") + + print("sleeping for some time before checking again..."); + + time.sleep(1) diff --git a/provision.bash b/provision.bash new file mode 100644 index 0000000..c0f7104 --- /dev/null +++ b/provision.bash @@ -0,0 +1,31 @@ +#!/bin/bash + +## + +set -e + +set -x + +## + +reset + +clear + +## + +echo "##" +echo "## launch > python jms client (aventador JMS)" +echo "##" + +sudo apt update; + +sudo apt install pip -y + +pip install stomp.py + +pip install selenium + +## for rest situations + +pip install requests