serious progress

DiskUtils.py

@@ -3,25 +3,54 @@ import time
 import re
 import fnmatch
 
-def dowork():
+def check_download_exists_matching_url_version_one(url):
 
-    ################################################################################################
+    print ("enter > downloads > exist check > version 1")
 
-    url = "https://sullygnome.com/channel/kaicenat/2023january/streams"
+    match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
 
-    print("checking disk for the stuff...")
+    if not match:
 
-    print("url", url)
+        print(f"Invalid URL format. Could not extract channel name, year, or month.")
 
-    fileExists = check_download_exists_matching_url(url)
+        return
 
-    print("file already exists / ",fileExists)
+    channel_name, year, month = match.groups()
 
-    ################################################################################################
+    print("channel / ",channel_name)
 
-def check_download_exists_matching_url(url):
+    #print("year / ",year)
 
-    print ("hello?")
+    #print("month / ",month)
+
+    #file_pattern = f"{channel_name}*{month}*{year}*.csv"
+
+    ## this is a problem. it works but if there 2 channels. 
+    ## redbull and redbull2 this is throw false positives. ugh
+
+    file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
+
+    for filename in os.listdir("/home/yankee/Downloads/"):
+
+        updated = filename.lower()
+
+        #print("file in downloads / ",updated)
+
+        if fnmatch.fnmatch(updated, file_pattern):
+
+            print(f"Found matching file: {updated}")
+
+            return True
+
+    else:
+
+        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'. (version one)")
+
+        return False
+
+def check_download_exists_matching_url_version_two(url):
+
+    print ("enter > check_download_exists_matching_url_version_two")
 
     match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
 
@@ -39,31 +68,69 @@ def check_download_exists_matching_url(url):
 
     print("month / ",month)
 
-    #file_pattern = f"{channel_name}*{month}*{year}*.csv"
+    for filename in os.listdir("/home/yankee/Downloads/"):
-
-    ## this is a problem. it works but if there 2 channels. 
-    ## redbull and redbull2 this is throw false positives. ugh
-
-    file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
-
-    for filename in os.listdir("/root/Downloads/"):
 
         updated = filename.lower()
 
-        print("file in downloads / ",updated)
+        print("found file (v2) / ",updated)
 
-        if fnmatch.fnmatch(updated, file_pattern):
+        ##
 
-            print(f"Found matching file: {updated}")
+        hasKeyDateParts = does_filename_contain_key_date_parts(updated, month, year)
 
-            return True
+        print("\thasKeyDateParts / ",hasKeyDateParts)
 
-    else:
+        ##
 
-        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'.")
+        hasTargetChannelName = does_filename_contain_target_channel_name(updated,channel_name)
+
+        print("\thasTargetChannelName / ",hasTargetChannelName)
+
+        ##
+
+    print ("exit < check_download_exists_matching_url_version_two")
+
+def does_filename_contain_target_channel_name(url,channel):  
+
+    regex = rf"\({re.escape(channel)}\)"
+
+    match = re.search(regex, url)
+
+    results = bool(match)
+
+    print("results / ",results)
+
+
+    match = re.search("\({channel}\)", url)
+
+    if match:
+        print("yup!")
+        extracted_text = match.group(1).lower()
+
+        print("extracted: /",extracted_text)
 
         return False
 
-def shutDownRobot():
+	##        return extracted_text == target_string.lower()
+
+    else:
+        print("nope!")
+        return False
+
+
+
+
+
+
+def does_filename_contain_key_date_parts(filename, monthString, yearNumber):
+
+    print("enter -> does_filename_contain_key_date_parts()")
+
+    regex = rf"{re.escape(monthString)} {yearNumber}"
+
+    doesContain = bool(re.search(regex, filename, re.IGNORECASE))
+
+    print("doesContain / ",doesContain)
+
+    return doesContain
 
-    print("shutting down the robot now")

RestAPIClient.py

@@ -22,7 +22,6 @@ def ip():
 
         return None
 
-
 def hostname():
 
     try:
@@ -56,13 +55,15 @@ def addDownloadSuccess(url):
     print(response.text)
 
 
-def addDownloadFailure(url):
+def addDownloadFailure(url, error):
 
     print("enter -> addDownloadFailure()")
 
     print("param / url / ",url)
 
-    data = {"url": url, "hostIP": ip(),"ip": ip(),"hostName":hostname()}
+    print("param / error / ",error)
+    
+    data = {"url": url, "hostIP": ip(),"ip": ip(),"hostName":hostname(), "reason":error}
 
     json_data = json.dumps(data)
 
@@ -78,9 +79,9 @@ def addDownloadFailure(url):
 
 def searchURLSuccesses(url):
 
-    print("enter -> callGET()")
+    ## print("enter -> callGET()")
 
-    print("param / url / ",url)
+    ## print("param / url / ",url)
     
     data = { "url": url }
 
@@ -90,15 +91,15 @@ def searchURLSuccesses(url):
 
     response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/success/exists", data=json_data, headers=headers)
 
-    print(response.text)
+    ## print(response.text)
 
     return response.text
 
 def searchURLFailures(url):
 
-    print("enter -> callGET()")
+    ## print("enter -> callGET()")
 
-    print("param / url / ",url)
+    ## print("param / url / ",url)
     
     data = { "url": url }
 
@@ -108,6 +109,6 @@ def searchURLFailures(url):
 
     response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/failure/exists", data=json_data, headers=headers)
 
-    print(response.text)
+    ## print(response.text)
 
     return response.text

SullyGnomeRobot.py

@@ -10,14 +10,12 @@ import RestAPIClient
 import DiskUtils
 from selenium.webdriver import FirefoxOptions
 
-## https://sullygnome.com/channel/kaicenat/365/games"
-
 def download(url):
 
     print(f"launch -> SullyGnomeRobot -> download() -> {url}")
 
-
     opts = FirefoxOptions()
+
     opts.add_argument("--headless")
 
     driver=webdriver.Firefox(options=opts)
@@ -28,7 +26,6 @@ def download(url):
 
     print("url / ", driver.current_url)
 
-
     ##########################################
 
     print("stage / start / tblControl_length check")
@@ -49,7 +46,7 @@ def download(url):
     
         print(f"Element with class name 'tblControl_length' not found: {e}")
         
-        RestAPIClient.addDownloadFailure(url)
+        RestAPIClient.addDownloadFailure(url, "Element with class name 'tblControl_length' not found")
 
         driver.quit()
     
@@ -75,7 +72,7 @@ def download(url):
 
         print(f"Element with class name 'TableExportLinkButton' not found: {e}")
 
-        RestAPIClient.addDownloadFailure(url)
+        RestAPIClient.addDownloadFailure(url,"Element with class name 'TableExportLinkButton' not found")
 
         driver.quit()
     
@@ -87,17 +84,47 @@ def download(url):
 
     print("file download started. now checking if exist before closing")
 
+    counter = 0
+
+    max_iterations = 5
+
     while True:
 
+        print(f"Loop iteration: {counter}")
+
+        counter += 1
+
+        if counter == max_iterations:
+
+            print(f"Loop exited after {max_iterations} iterations.")
+
+            print("shutting down the robot")
+
+            driver.quit()
+
+            print("robot shut down. breaking. and dying.")
+
+            RestAPIClient.addDownloadFailure(url,"file not found on disk")
+
+            print("stage / complete / database updated w/ FAILURE")
+
+            break
+
         print("checking is file exists... [loop]")
 
-        existsAlready = DiskUtils.check_download_exists_matching_url(url)
+        existsAlreadyVersionOne = DiskUtils.check_download_exists_matching_url_version_one(url)
 
-        print("file already exists / ",existsAlready)
+        print("file already exists (v1) / ",existsAlreadyVersionOne)
 
-        if existsAlready:
+        existsAlreadyVersionTwo = DiskUtils.check_download_exists_matching_url_version_two(url)
 
-            print(f"The file matching the url '{url}' exists.")
+        print("file already exists (v2) / ",existsAlreadyVersionTwo)
+
+        time.sleep(40)
+
+        if existsAlreadyVersionOne:
+
+            print(f"The file matching the url '{url}' DOES INDEED exists. (version 1)")
 
             print("sleeping for 4 secs before shutting down robot")
 
@@ -109,6 +136,10 @@ def download(url):
 
             print("robot shut down. breaking. and dying.")
 
+            RestAPIClient.addDownloadSuccess(url)
+
+            print("stage / complete / database updated w/ success")
+
             break
 
         else:
@@ -123,8 +154,4 @@ def download(url):
 
     ##########################################
 
-    RestAPIClient.addDownloadSuccess(url)
-
-    print("stage / complete / database updated w/ success")
-
     print(f"complete -> SullyGnomeRobot -> download() -> {url}")

consumer.bash

@@ -3,15 +3,13 @@
 set -e
 
 set -x
+
 ##
 
-echo "what up????"
+##cd /yankee-gnome-fire-consumer
 
-cd /yankee-gnome-fire-consumer
+echo "##"
+echo "## init > yankee > consumer"
+echo "##"
 
-pwd 
-
-ls
-
-whoami
 python3 consumer.py

consumer.py

@@ -3,43 +3,80 @@ import time
 import SullyGnomeRobot
 import RestAPIClient
 
+### start variables ###
+
+jmsQueue="asian-tester"
+
+host = '38.107.226.34'
+
+port = 61613
+
+
+username = 'admin'
+password = 'password'
+
+
+### end variables ###
+
 print("## ")
-print("## starting > JMS consumer (yankee-sully-channels-monthly)")
+print(f"## starting > JMS consumer app!")
 print("## ")
 
+def writeSullyURLToJMS():
+
+    channel="https://sullygnome.com/channel/yaritaiji/2024october/games"
+    
+    print(f"launch -> jms writer -> channel-> {channel}")
+
+    conn = stomp.Connection([('osiris.yankee.embanet.online', 61613)])
+
+    conn.connect('admin', 'password', wait=True)
+
+    try:
+
+        conn.send(body=channel.strip(), destination=f'/queue/{jmsQueue}')
+
+    finally:
+
+        conn.disconnect()
+        
 class MyListener(stomp.ConnectionListener):
 
     def on_message(headers, message):
 
-        print("** new message! **")
+        ## print("** new message! **")
 
-        print("message / ", message)
+        ## print("message / ", message)
 
-        print("messagebody / ", message.body)
+        print("JMS url -> ", message.body)
 
-        print("checking the value does or doesnt exist in db")
+        ## print("checking the value does or doesnt exist in db")
 
         alreadyExistsSuccess = RestAPIClient.searchURLSuccesses(message.body)
 
-        print("alreadyExistsSuccess / ", alreadyExistsSuccess)
+        ## print("alreadyExistsSuccess / ", alreadyExistsSuccess)
 
         if alreadyExistsSuccess.lower().strip() == "true":
     
-            print("this record exists (successes). returning. doing nothing")
+            ## print("this record exists (successes). returning. doing nothing")
 
-            return
+            ## return
 
-        print("didn't exist in success. check in errors now.")
+            print ("skipped!!")
+            
+        ## print("didn't exist in success. check in errors now.")
 
         alreadyExistsFailure = RestAPIClient.searchURLFailures(message.body)
 
-        print("alreadyExistsFailure / ", alreadyExistsFailure)
+        ## print("alreadyExistsFailure / ", alreadyExistsFailure)
 
         if alreadyExistsFailure.lower().strip() == "true":
 
-            print("this record exists (failures). returning. doing nothing")
+            ## print("this record exists (failures). returning. doing nothing")
 
-            return
+            ##return
+            
+            print ("skipped!!")
 
         print("didn't exist in success or errors. will process now")
 
@@ -51,28 +88,33 @@ class MyListener(stomp.ConnectionListener):
 
             print(f"exception / something went wrong when the sully gnome robot was downloading: {e}")
 
-            RestAPIClient.addDownloadFailure(message.body)
+            RestAPIClient.addDownloadFailure(message.body, e)
 
     def on_error(headers, message):
 
         print('received an error "%s"' % message)
         
-        RestAPIClient.addDownloadFailure(message.body)
+        RestAPIClient.addDownloadFailure(message.body, "error from onmessage (consumer.py)")
 
 print("## ")
-print("## starting > JMS consumer (yankee-sully-channels-monthly)")
+print(f"## starting > JMS consumer ([{jmsQueue}]")
 print("## ")
 
-conn = stomp.Connection([('67.220.70.106', 61613)])
+writeSullyURLToJMS()
+connection_details = [(host, port)]
+
+conn = stomp.Connection(connection_details)
 
 conn.set_listener('', MyListener())
 
-conn.connect('admin', 'password', wait=True)
+conn.connect(username, password, wait=True)
 
-conn.subscribe('/queue/yankee-sully-channels-monthly', id=1, ack='auto', headers={'activemq.prefetchSize': 10})
+conn.subscribe(f'/queue/{jmsQueue}', id=1, ack='auto', headers={'activemq.prefetchSize': 10})
+
+print(f"Subscribed to queue: {jmsQueue}")
     
 print("## ")
-print("## notification > connected to JMS server (yankee-sully-channels-monthly) 3333")
+print(f"## notification > connected to JMS server [{jmsQueue}] (upgrade)")
 print("## ")
 
 while True:
@@ -82,3 +124,4 @@ while True:
     time.sleep(1)
        
 conn.disconnect()
+

recycle/crontabX

@@ -0,0 +1,6 @@
+SHELL=/bin/bash
+
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+6 11 * * * python3 /yankee-gnome-fire-consumer/consumer.py >> /tmp/root/Downloads/.logs/yankee-node-96.255.165.82-veneno.log 2>&1
+

recycle/crontabX2

@@ -0,0 +1,7 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+48 20 * * *  /yankee-gnome-fire-consumer/consumer.bash >> /var/log/yankee.log
+
+* * * * * /yankee-gnome-fire-consumer/ping.bash >> /var/log/cron.log 2>&1
+

recycle/ping.bash

@@ -0,0 +1,4 @@
+
+touch /tmp/`date +%Y_%m_%d_%H_%M_%S`.dat
+
+ls -lha /tmp/*dat

recycle/smart-cron-builder (copy 1).py

@@ -0,0 +1,151 @@
+import datetime
+import subprocess
+import socket
+import os
+import requests
+
+def buildLogDirectory():
+
+    return "/root/Downloads/.logs/"
+
+def buildLogFileName():
+
+    logFileDir = buildLogDirectory()
+
+    try:
+
+        response = requests.get("https://api.ipify.org?format=json")  # Or another similar service
+
+        response.raise_for_status()  # Check for HTTP errors (4xx or 5xx)
+
+        ip_data = response.json()
+
+        server_ip = ip_data['ip']
+
+        print("server_ip / ",ip_data['ip'])
+
+        server_hostname = socket.gethostname()
+
+        print("server_hostname / ",server_hostname)
+
+        node_id= logFileDir + "yankee-node-" + server_ip + "-" + server_hostname + ".log"
+
+        return node_id
+
+    except socket.gaierror:
+
+        return "!!error!!"
+
+def buildCronExecuteTime():
+
+    current_time = datetime.datetime.now()
+
+    new_minute = (current_time.minute + 2) % 60  # The % operator performs the wrap-around
+
+    print("new_minute / ",new_minute)
+
+    new_hour = current_time.hour + (current_time.minute + 5) // 60 # Integer division for hour increment
+
+    print("new_hour / ",new_hour)
+
+    cron = str(new_minute) + " " + str(new_hour) + " " + "* * *"
+
+    return cron
+
+def buildCronSituation():
+
+    return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"
+
+####
+
+cronTiming = buildCronExecuteTime()
+
+print("cronTiming / ",cronTiming)
+
+##
+
+logger = buildLogFileName();
+
+print("logger / ",logger)
+
+##
+
+cronFile = buildCronSituation();
+
+print()
+print(cronFile)
+print()
+
+
+
+def createLogDir():
+
+    log_dir = buildLogDirectory()
+
+    print("creating log dir",log_dir)
+
+    try:
+        if not os.path.exists(log_dir):  # Check if the directory exists
+            os.makedirs(log_dir)  # Create the directory (and any necessary parent directories)
+            print(f"Directory '{log_dir}' created successfully.")
+        else:
+            print(f"Directory '{log_dir}' already exists.")
+        return True  # Directory exists or was created successfully
+
+    except OSError as e:
+        print(f"Error creating directory '{log_dir}': {e}")
+        return False
+
+##
+
+
+def writeCrontabSituation():
+
+    file_path ="crontabX"
+
+    crontab_text = """
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+* * * * * python3 /yankee-gnome-fire-consumer/consumer.py >> /var/log/yankee-downloader.log 2>&1
+
+* * * * * /tmp/ping2.bash >> /var/log/cron.log 2>&1"""
+
+    try:
+        # Check if the file exists and delete if it does
+        if os.path.exists(file_path):
+            os.remove(file_path)
+
+        with open(file_path, "w") as f:
+
+            sep="\n"
+
+            f.write("SHELL=/bin/bash")
+            f.write(sep)
+
+            f.write("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
+            f.write(sep)
+            f.write(sep)
+
+            f.write(buildCronSituation())
+            f.write(sep)
+            f.write(sep)
+
+            f.write("* * * * * /tmp/ping.bash >> /var/log/cron.log 2>&1")
+            f.write(sep)
+            f.write(sep)
+
+        print(f"Crontab written to '{file_path}' successfully.")
+
+        return True
+
+    except OSError as e:
+
+        print(f"Error writing to file '{file_path}': {e}")
+
+        return False
+
+####################
+
+createLogDir()
+
+writeCrontabSituation()

recycle/smart-cron-builder.bash

@@ -0,0 +1,17 @@
+reset
+
+clear
+
+python3 smart-cron-builder.py
+
+####
+
+cp crontabX /etc/cron.d/cronfile
+
+# Give execution rights on the cron job
+chmod 0644 /etc/cron.d/cronfile
+
+# Apply the cron job
+crontab -u root /etc/cron.d/cronfile
+
+crontab -l

recycle/smart-cron-builder.py

@@ -0,0 +1,157 @@
+import datetime
+import subprocess
+import socket
+import os
+import requests
+
+def buildLogDirectory():
+
+    return "/root/Downloads/.logs/"
+
+def buildLogFileName():
+
+    logFileDir = buildLogDirectory()
+
+    try:
+
+        response = requests.get("https://api.ipify.org?format=json")  # Or another similar service
+
+        response.raise_for_status()  # Check for HTTP errors (4xx or 5xx)
+
+        ip_data = response.json()
+
+        server_ip = ip_data['ip']
+
+        print("server_ip / ",ip_data['ip'])
+
+        server_hostname = socket.gethostname()
+
+        print("server_hostname / ",server_hostname)
+
+        #node_id=logFileDir + "yankee-node-" + server_ip + "-" + server_hostname + ".log"
+
+        node_id= "/var/log/yankee.log"
+
+        return node_id
+
+    except socket.gaierror:
+
+        return "!!error!!"
+
+def buildCronExecuteTime():
+
+    current_time = datetime.datetime.now()
+
+    new_minute = (current_time.minute + 2) % 60
+
+    print("new_minute / ",new_minute)
+
+    new_hour = (current_time.hour + 5) % 24
+
+    print("new_hour / ",new_hour)
+
+    cron = str(new_minute) + " " + str(new_hour) + " " + "* * * "
+
+    return cron
+
+def buildCronSituation():
+
+    ##return "* * * * * /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
+
+    return buildCronExecuteTime() + " /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
+
+    ##return buildCronExecuteTime() +" /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName() + " 2>&1"
+
+    ##return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"
+
+####
+
+cronTiming = buildCronExecuteTime()
+
+print("cronTiming / ",cronTiming)
+
+##
+
+logger = buildLogFileName();
+
+print("logger / ",logger)
+
+##
+
+cronFile = buildCronSituation();
+
+print()
+print(cronFile)
+print()
+
+
+
+def createLogDir():
+
+    log_dir = buildLogDirectory()
+
+    print("creating log dir",log_dir)
+
+    try:
+        if not os.path.exists(log_dir):  # Check if the directory exists
+            os.makedirs(log_dir)  # Create the directory (and any necessary parent directories)
+            print(f"Directory '{log_dir}' created successfully.")
+        else:
+            print(f"Directory '{log_dir}' already exists.")
+        return True  # Directory exists or was created successfully
+
+    except OSError as e:
+        print(f"Error creating directory '{log_dir}': {e}")
+        return False
+
+##
+
+
+def writeCrontabSituation():
+
+    file_path ="crontabX"
+
+    try:
+
+        if os.path.exists(file_path):
+
+            os.remove(file_path)
+
+        with open(file_path, "w") as f:
+
+            sep="\n"
+
+            f.write("SHELL=/bin/bash")
+            f.write(sep)
+
+            f.write("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
+            f.write(sep)
+            f.write(sep)
+
+            f.write(buildCronSituation())
+            f.write(sep)
+            f.write(sep)
+
+            f.write("* * * * * /yankee-gnome-fire-consumer/ping.bash >> /var/log/cron.log 2>&1")
+            f.write(sep)
+            f.write(sep)
+
+            #worksf.write("* * * * * /yankee-gnome-fire-consumer/consumer.bash >> /var/log/consumer.log 2>&1")
+            #f.write(sep)
+            #f.write(sep)
+
+        print(f"Crontab written to '{file_path}' successfully.")
+
+        return True
+
+    except OSError as e:
+
+        print(f"Error writing to file '{file_path}': {e}")
+
+        return False
+
+####################
+
+#createLogDir()
+
+writeCrontabSituation()