automated terminal push

.gitignore

@@ -0,0 +1,4 @@
+__pycache__
+
+Downloads
+

.trash/README.md

@@ -0,0 +1,2 @@
+# yankee-fire-control-code
+

.trash/RestAPIClient (copy 1).py

@@ -0,0 +1,37 @@
+import requests
+
+## https://www.nylas.com/blog/use-python-requests-module-rest-apis/#how-to-use-python-requests
+
+def callGET(url):
+
+    print("enter -> callGET()")
+
+    print("param / url / ",url)
+    
+    params = dict(
+        origin='Chicago,IL',
+        destination='Los+Angeles,CA',
+        waypoints='Joplin,MO|Oklahoma+City,OK',
+        sensor='false',
+        url2=url
+    )
+
+    headers = {'Accept': 'application/json'}
+
+    print("params", params)
+
+    response = requests.post("https://osiris.yankee.valorantdigital.com/channel/monthly/games/success/exists", headers=headers, data=params, params=params)
+    
+    print("response2", response.json())
+
+def callPOST():
+
+    print("enter -> callPOST()")
+    
+    response = requests.post('https://httpbin.org/post', data = {'key':'value'})
+    
+    print("response", response.json())
+
+callGET("https://embanet.online")
+
+#callPOST()

.trash/jmsclient.py

@@ -0,0 +1,42 @@
+import stomp
+import time
+import veneno
+import fetcher
+
+class MyListener(stomp.ConnectionListener):
+
+    def on_message(headers, message):
+
+        conn.ack(headers['message-id'])
+
+        print("** new message! **")
+
+        print(message)
+
+        fetcher.fetch(message.body)
+
+
+
+    def on_error(headers, message):
+
+        print('received an error "%s"' % message)
+        
+        conn.ack(headers['message-id'])
+
+conn = stomp.Connection([('osiris.yankee.valorantdigital.com', 61613)])
+
+conn.set_listener('', MyListener())
+
+conn.connect('admin', 'password', wait=True)
+
+conn.subscribe('/queue/sully-gnome-urls', id=1, ack='client', headers={'activemq.prefetchSize': 10})
+
+print("## notification > connected to JMS server")
+
+while True:
+
+    pass
+
+    time.sleep(1)
+    
+conn.disconnect()

.trash/veneno.py

@@ -0,0 +1,19 @@
+# veneno.py
+
+def fetch(x):
+
+  print(f"launch -> data fetcher -> url -> {x}")
+
+def eat(x,y):
+
+  """Consumes x and y (pretend eating!).
+
+  Args:
+      x: The first amount to be consumed.
+      y: The second amount to be consumed.
+
+  Returns:
+      None
+  """
+
+  print(f"Eating {x} and {y}...")

DiskUtils.py

@@ -0,0 +1,69 @@
+import os
+import time
+import re
+import fnmatch
+
+def dowork():
+
+    ################################################################################################
+
+    url = "https://sullygnome.com/channel/kaicenat/2023january/streams"
+
+    print("checking disk for the stuff...")
+
+    print("url", url)
+
+    fileExists = check_download_exists_matching_url(url)
+
+    print("file already exists / ",fileExists)
+
+    ################################################################################################
+
+def check_download_exists_matching_url(url):
+
+    print ("hello?")
+
+    match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
+
+    if not match:
+
+        print(f"Invalid URL format. Could not extract channel name, year, or month.")
+
+        return
+
+    channel_name, year, month = match.groups()
+
+    print("channel / ",channel_name)
+
+    print("year / ",year)
+
+    print("month / ",month)
+
+    #file_pattern = f"{channel_name}*{month}*{year}*.csv"
+
+    ## this is a problem. it works but if there 2 channels. 
+    ## redbull and redbull2 this is throw false positives. ugh
+
+    file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
+
+    for filename in os.listdir("/home/yankee/Downloads/"):
+
+        updated = filename.lower()
+
+        print("file in downloads / ",updated)
+
+        if fnmatch.fnmatch(updated, file_pattern):
+
+            print(f"Found matching file: {updated}")
+
+            return True
+
+    else:
+
+        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'.")
+
+        return False
+
+def shutDownRobot():
+
+    print("shutting down the robot now")

RestAPIClient.py

@@ -0,0 +1,146 @@
+import requests
+import json
+import socket
+
+## https://www.nylas.com/blog/use-python-requests-module-rest-apis/#how-to-use-python-requests
+## https://www.abstractapi.com/guides/ip-geolocation/get-ip-address-python
+
+def ip():
+
+    try:
+
+        public_ip = requests.get('https://api.ipify.org').text
+
+        print(f"Public IP Address: {public_ip}")
+
+        return public_ip
+
+    except requests.RequestException as e:
+
+        print(f"Error retrieving public IP address: {e}")
+
+        return None
+
+def addDownloadSuccess(url):
+
+    print("enter -> addDownloadSuccess()")
+
+    print("param / url / ",url)
+    
+    data = {"url": url, "hostIP": ip(),"ip": ip(),}
+
+    json_data = json.dumps(data)
+
+    print("param / data / ",data)
+
+    print("param / data / ",json_data)
+
+    headers = {"Content-Type": "application/json"}
+
+
+
+    response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/success/add", data=json_data, headers=headers)
+
+    print(response.text)
+
+
+def addDownloadFailure(url):
+
+    print("enter -> addDownloadFailure()")
+
+    print("param / url / ",url)
+    
+    data = {"url": url, "hostIP": ip(),"ip": ip(),}
+
+    json_data = json.dumps(data)
+
+    print("param / data / ",data)
+
+    print("param / data / ",json_data)
+
+    headers = {"Content-Type": "application/json"}
+
+
+
+    response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/failure/add", data=json_data, headers=headers)
+
+    print(response.text)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+    #print("params", params)
+
+    #response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/success/exists", headers=headers, data=params, params=params)
+    
+    #print("response2", response.json())
+
+
+def searchURLSuccesses(url):
+
+    print("enter -> callGET()")
+
+    print("param / url / ",url)
+    
+    params = dict(
+        origin='Chicago,IL',
+        destination='Los+Angeles,CA',
+        waypoints='Joplin,MO|Oklahoma+City,OK',
+        sensor='false',
+        url2=url
+    )
+
+
+    #data = {"name": "Jane Smith", "email": "janesmith@example.com", "url": url }
+
+    data = { "url": url }
+
+    headers = {"Content-Type": "application/json"}
+
+    json_data = json.dumps(data)
+
+    response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/success/exists", data=json_data, headers=headers)
+
+    print(response.text)
+
+    return response.text
+
+def searchURLFailures(url):
+
+    print("enter -> callGET()")
+
+    print("param / url / ",url)
+    
+    params = dict(
+        origin='Chicago,IL',
+        destination='Los+Angeles,CA',
+        waypoints='Joplin,MO|Oklahoma+City,OK',
+        sensor='false',
+        url2=url
+    )
+
+
+    #data = {"name": "Jane Smith", "email": "janesmith@example.com", "url": url }
+
+    data = { "url": url }
+
+    headers = {"Content-Type": "application/json"}
+
+    json_data = json.dumps(data)
+
+    response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/failure/exists", data=json_data, headers=headers)
+
+    print(response.text)
+
+    return response.text

SullyGnomeRobot.py

@@ -0,0 +1,120 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import time
+from selenium.webdriver.support.select import Select
+import RestAPIClient
+import DiskUtils
+
+## https://sullygnome.com/channel/kaicenat/365/games"
+
+def download(url):
+
+    print(f"launch -> SullyGnomeRobot -> download() -> {url}")
+
+    driver = webdriver.Firefox()
+
+    driver.get(url)
+
+    ##########################################
+
+    print("stage / start / tblControl_length check")
+
+    try:
+    
+        dataset_drop_down_element = WebDriverWait(driver, 5).until(
+
+            EC.presence_of_element_located((By.NAME, "tblControl_length"))
+
+        )
+
+        dataset_drop_down_element = Select(dataset_drop_down_element)
+
+        dataset_drop_down_element.select_by_visible_text("100")
+
+    except Exception as e:
+    
+        print(f"Element with class name 'tblControl_length' not found: {e}")
+        
+        RestAPIClient.addDownloadFailure(url)
+
+        driver.quit()
+    
+        return False 
+        
+    print("stage / complete / tblControl_length check")
+
+    ##########################################
+
+    print("stage / start / TableExportLinkButton check")
+
+    try:
+    
+        WebDriverWait(driver, 5).until(
+        EC.presence_of_element_located((By.CLASS_NAME, "TableExportLinkButton"))
+        )
+        
+        link = driver.find_element(By.CLASS_NAME, "TableExportLinkButton")
+
+        link.click()
+
+    except Exception as e:
+
+        print(f"Element with class name 'TableExportLinkButton' not found: {e}")
+
+        RestAPIClient.addDownloadFailure(url)
+
+        driver.quit()
+    
+        return False 
+
+    print("stage / complete / TableExportLinkButton check")
+
+    ##########################################
+
+    print("file download started. now checking if exist before closing")
+
+    while True:
+
+        print("checking is file exists... [loop]")
+
+        existsAlready = DiskUtils.check_download_exists_matching_url(url)
+
+        print("file already exists / ",existsAlready)
+
+        if existsAlready:
+
+            print(f"The file matching the url '{url}' exists.")
+
+            print("sleeping for 4 secs before shutting down robot")
+
+            time.sleep(4)
+
+            print("shutting down the robot")
+
+            driver.quit()
+
+            print("robot shut down. breaking. and dying.")
+
+            break
+
+        else:
+
+            print(f"The file matching the url '{url}' DOES NOT exist.")
+
+            print("sleeping for some time before checking again..."); 
+
+            time.sleep(1)
+
+    print(" file is finished downloading.moving on.")
+
+    ##########################################
+
+    RestAPIClient.addDownloadSuccess(url)
+
+    print("stage / complete / database updated w/ success")
+
+    print(f"complete -> SullyGnomeRobot -> download() -> {url}")

consumer.bash

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+##
+
+set -e
+
+set -x
+
+##
+
+reset
+
+clear
+
+##
+
+codeDir=/home/yankee/Code
+
+rm -rf $codeDir
+
+mkdir $codeDir
+
+ln -s /yankee/code $codeDir
+
+ls -lha $codeDir
+
+##
+
+downloadsDir=/home/yankee/Downloads
+
+rm -rf /home/yankee/Downloads
+
+##mkdir $downloadsDir
+
+ln -s /yankee/downloads/ /home/yankee/Downloads
+
+ls -lha $downloadsDir
+
+##
+
+echo "##"
+echo "## launch > yankee gnome > consumer.py"
+echo "##"
+
+python3 consumer.py
+

consumer.py

@@ -0,0 +1,76 @@
+import stomp
+import time
+import SullyGnomeRobot
+import RestAPIClient
+
+class MyListener(stomp.ConnectionListener):
+
+    def on_message(headers, message):
+
+        print("** new message! **")
+
+        print("message / ", message)
+
+        print("messagebody / ", message.body)
+
+        print("checking the value does or doesnt exist in db")
+
+        alreadyExistsSuccess = RestAPIClient.searchURLSuccesses(message.body)
+
+        print("alreadyExistsSuccess / ", alreadyExistsSuccess)
+
+        if alreadyExistsSuccess.lower().strip() == "true":
+    
+            print("this record exists (successes). returning. doing nothing")
+
+            return
+
+        print("didn't exist in success. check in errors now.")
+
+        alreadyExistsFailure = RestAPIClient.searchURLFailures(message.body)
+
+        print("alreadyExistsFailure / ", alreadyExistsFailure)
+
+        if alreadyExistsFailure.lower().strip() == "true":
+
+            print("this record exists (failures). returning. doing nothing")
+
+            return
+
+        print("didn't exist in success or errors. will process now")
+
+        try:
+        
+            SullyGnomeRobot.download(message.body)
+
+        except Exception as e:
+
+            print(f"exception / something went wrong when the sully gnome robot was downloading: {e}")
+
+            RestAPIClient.addDownloadFailure(message.body)
+
+    def on_error(headers, message):
+
+        print('received an error "%s"' % message)
+        
+        RestAPIClient.addDownloadFailure(message.body)
+
+conn = stomp.Connection([('osiris.yankee.embanet.online', 61613)])
+
+conn.set_listener('', MyListener())
+
+conn.connect('admin', 'password', wait=True)
+
+conn.subscribe('/queue/yankee-sully-channels-monthly', id=1, ack='auto', headers={'activemq.prefetchSize': 10})
+
+print("## ")
+print("## notification > connected to JMS server (yankee-sully-channels-monthly)")
+print("## ")
+
+while True:
+
+    pass
+
+    time.sleep(1)
+    
+conn.disconnect()

downloadcheck/fastdownload.bash

@@ -0,0 +1,9 @@
+set -e
+
+set -x 
+
+reset
+
+clear
+
+python3 fastdownload.py

downloadcheck/fastdownload.py

@@ -0,0 +1,89 @@
+import os
+import time
+import re
+
+def dowork():
+
+    ################################################################################################
+
+    url = "https://sullygnome.com/channel/kaicenat/2023january/streams"
+
+    print("checking disk for the stuff...")
+
+    print("url", url)
+
+    fileExists = check_download_exists_matching_url(url)
+
+    print("file already exists / ",fileExists)
+
+    ################################################################################################
+
+def check_download_exists_matching_url(url):
+
+    print ("hello?")
+
+    match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/streams", url.lower())
+
+    if not match:
+
+        print(f"Invalid URL format. Could not extract channel name, year, or month.")
+
+        return
+
+    channel_name, year, month = match.groups()
+
+    print("channel / ",channel_name)
+
+    print("year / ",year)
+
+    print("month / ",month)
+
+    #file_pattern = f"{channel_name}*{month}*{year}*.csv"
+
+    file_pattern = f"{channel_name} - Twitch stream stats in {month} {year} - SullyGnome.csv".lower()
+
+    for filename in os.listdir("/home/softwareshinobi/Downloads/"):
+
+        updated = filename.lower()
+
+        print("updated / ",updated)
+
+        if re.match(file_pattern, updated):
+
+            print(f"Found matching file: {updated}")
+
+            return True
+
+    else:
+
+        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'.")
+
+        return False
+
+def shutDownRobot():
+
+    print("shutting down the robot now")
+
+while True:
+
+    print("checking is file exists... [loop]")
+
+    if check_file(file_to_check):
+
+        print(f"The file '{file_to_check}' exists.")
+
+        print("shutting down the robot")
+
+        shutDownRobot()
+
+        print("robot shut down. breaking. and dying.")
+
+        break
+
+    else:
+
+        print(f"The file '{file_to_check}' does not exist.")
+
+        print("sleeping for some time before checking again..."); 
+
+        time.sleep(1)

provision.bash

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+##
+
+set -e
+
+set -x
+
+##
+
+reset
+
+clear
+
+##
+
+echo "##"
+echo "## launch > python jms client (aventador JMS)"
+echo "##"
+
+sudo apt update;
+
+sudo apt install pip -y
+
+pip install stomp.py 
+
+pip install selenium
+
+## for rest situations
+
+pip install requests