crontab add

DiskUtils.py

@@ -3,54 +3,25 @@ import time
 import re
 import fnmatch
 
-def check_download_exists_matching_url_version_one(url):
+def dowork():
 
-    print ("enter > downloads > exist check > version 1")
+    ################################################################################################
 
-    match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
+    url = "https://sullygnome.com/channel/kaicenat/2023january/streams"
 
-    if not match:
+    print("checking disk for the stuff...")
 
-        print(f"Invalid URL format. Could not extract channel name, year, or month.")
+    print("url", url)
 
-        return
+    fileExists = check_download_exists_matching_url(url)
 
-    channel_name, year, month = match.groups()
+    print("file already exists / ",fileExists)
 
-    print("channel / ",channel_name)
+    ################################################################################################
 
-    #print("year / ",year)
+def check_download_exists_matching_url(url):
 
-    #print("month / ",month)
-
-    #file_pattern = f"{channel_name}*{month}*{year}*.csv"
-
-    ## this is a problem. it works but if there 2 channels. 
-    ## redbull and redbull2 this is throw false positives. ugh
-
-    file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
-
-    for filename in os.listdir("/home/yankee/Downloads/"):
-
-        updated = filename.lower()
-
-        #print("file in downloads / ",updated)
-
-        if fnmatch.fnmatch(updated, file_pattern):
-
-            print(f"Found matching file: {updated}")
-
-            return True
-
-    else:
-
-        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'. (version one)")
-
-        return False
-
-def check_download_exists_matching_url_version_two(url):
-
-    print ("enter > check_download_exists_matching_url_version_two")
+    print ("hello?")
 
     match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
 
@@ -68,69 +39,31 @@ def check_download_exists_matching_url_version_two(url):
 
     print("month / ",month)
 
-    for filename in os.listdir("/home/yankee/Downloads/"):
+    #file_pattern = f"{channel_name}*{month}*{year}*.csv"
+
+    ## this is a problem. it works but if there 2 channels. 
+    ## redbull and redbull2 this is throw false positives. ugh
+
+    file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
+
+    for filename in os.listdir("/root/Downloads/"):
 
         updated = filename.lower()
 
-        print("found file (v2) / ",updated)
+        print("file in downloads / ",updated)
 
-        ##
+        if fnmatch.fnmatch(updated, file_pattern):
 
-        hasKeyDateParts = does_filename_contain_key_date_parts(updated, month, year)
+            print(f"Found matching file: {updated}")
 
-        print("\thasKeyDateParts / ",hasKeyDateParts)
-
-        ##
-
-        hasTargetChannelName = does_filename_contain_target_channel_name(updated,channel_name)
-
-        print("\thasTargetChannelName / ",hasTargetChannelName)
-
-        ##
-
-    print ("exit < check_download_exists_matching_url_version_two")
-
-def does_filename_contain_target_channel_name(url,channel):  
-
-    regex = rf"\({re.escape(channel)}\)"
-
-    match = re.search(regex, url)
-
-    results = bool(match)
-
-    print("results / ",results)
-
-
-    match = re.search("\({channel}\)", url)
-
-    if match:
-        print("yup!")
-        extracted_text = match.group(1).lower()
-
-        print("extracted: /",extracted_text)
-
-        return False
-
-	##        return extracted_text == target_string.lower()
+            return True
 
     else:
-        print("nope!")
+
+        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'.")
+
         return False
 
+def shutDownRobot():
 
-
-
-
-
-def does_filename_contain_key_date_parts(filename, monthString, yearNumber):
-
-    print("enter -> does_filename_contain_key_date_parts()")
-
-    regex = rf"{re.escape(monthString)} {yearNumber}"
-
-    doesContain = bool(re.search(regex, filename, re.IGNORECASE))
-
-    print("doesContain / ",doesContain)
-
-    return doesContain
-
+    print("shutting down the robot now")

RestAPIClient.py

@@ -79,9 +79,9 @@ def addDownloadFailure(url, error):
 
 def searchURLSuccesses(url):
 
-    ## print("enter -> callGET()")
+    print("enter -> callGET()")
 
-    ## print("param / url / ",url)
+    print("param / url / ",url)
     
     data = { "url": url }
 
@@ -91,15 +91,15 @@ def searchURLSuccesses(url):
 
     response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/success/exists", data=json_data, headers=headers)
 
-    ## print(response.text)
+    print(response.text)
 
     return response.text
 
 def searchURLFailures(url):
 
-    ## print("enter -> callGET()")
+    print("enter -> callGET()")
 
-    ## print("param / url / ",url)
+    print("param / url / ",url)
     
     data = { "url": url }
 
@@ -109,6 +109,6 @@ def searchURLFailures(url):
 
     response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/failure/exists", data=json_data, headers=headers)
 
-    ## print(response.text)
+    print(response.text)
 
     return response.text

SullyGnomeRobot.py

@@ -10,6 +10,8 @@ import RestAPIClient
 import DiskUtils
 from selenium.webdriver import FirefoxOptions
 
+## https://sullygnome.com/channel/kaicenat/365/games"
+
 def download(url):
 
     print(f"launch -> SullyGnomeRobot -> download() -> {url}")
@@ -26,6 +28,7 @@ def download(url):
 
     print("url / ", driver.current_url)
 
+
     ##########################################
 
     print("stage / start / tblControl_length check")
@@ -84,47 +87,17 @@ def download(url):
 
     print("file download started. now checking if exist before closing")
 
-    counter = 0
-
-    max_iterations = 5
-
     while True:
 
-        print(f"Loop iteration: {counter}")
-
-        counter += 1
-
-        if counter == max_iterations:
-
-            print(f"Loop exited after {max_iterations} iterations.")
-
-            print("shutting down the robot")
-
-            driver.quit()
-
-            print("robot shut down. breaking. and dying.")
-
-            RestAPIClient.addDownloadFailure(url,"file not found on disk")
-
-            print("stage / complete / database updated w/ FAILURE")
-
-            break
-
         print("checking is file exists... [loop]")
 
-        existsAlreadyVersionOne = DiskUtils.check_download_exists_matching_url_version_one(url)
+        existsAlready = DiskUtils.check_download_exists_matching_url(url)
 
-        print("file already exists (v1) / ",existsAlreadyVersionOne)
+        print("file already exists / ",existsAlready)
 
-        existsAlreadyVersionTwo = DiskUtils.check_download_exists_matching_url_version_two(url)
+        if existsAlready:
 
-        print("file already exists (v2) / ",existsAlreadyVersionTwo)
-
-        time.sleep(40)
-
-        if existsAlreadyVersionOne:
-
-            print(f"The file matching the url '{url}' DOES INDEED exists. (version 1)")
+            print(f"The file matching the url '{url}' exists.")
 
             print("sleeping for 4 secs before shutting down robot")
 
@@ -136,10 +109,6 @@ def download(url):
 
             print("robot shut down. breaking. and dying.")
 
-            RestAPIClient.addDownloadSuccess(url)
-
-            print("stage / complete / database updated w/ success")
-
             break
 
         else:
@@ -154,4 +123,8 @@ def download(url):
 
     ##########################################
 
+    RestAPIClient.addDownloadSuccess(url)
+
+    print("stage / complete / database updated w/ success")
+
     print(f"complete -> SullyGnomeRobot -> download() -> {url}")

consumer.bash

@@ -6,10 +6,10 @@ set -x
 
 ##
 
-##cd /yankee-gnome-fire-consumer
+echo "init > consumer.bash"
 
-echo "##"
-echo "## init > yankee > consumer"
-echo "##"
+####
+
+cd /yankee-gnome-fire-consumer
 
 python3 consumer.py

consumer.py

@@ -2,81 +2,46 @@ import stomp
 import time
 import SullyGnomeRobot
 import RestAPIClient
-
-### start variables ###
-
-jmsQueue="asian-tester"
-
-host = '38.107.226.34'
-
-port = 61613
-
-
-username = 'admin'
-password = 'password'
-
-
-### end variables ###
+import os
+import signal
 
 print("## ")
-print(f"## starting > JMS consumer app!")
+print("## starting > JMS consumer (yankee-sully-channels-monthly)")
 print("## ")
 
-def writeSullyURLToJMS():
-
-    channel="https://sullygnome.com/channel/yaritaiji/2024october/games"
-    
-    print(f"launch -> jms writer -> channel-> {channel}")
-
-    conn = stomp.Connection([('osiris.yankee.embanet.online', 61613)])
-
-    conn.connect('admin', 'password', wait=True)
-
-    try:
-
-        conn.send(body=channel.strip(), destination=f'/queue/{jmsQueue}')
-
-    finally:
-
-        conn.disconnect()
-        
 class MyListener(stomp.ConnectionListener):
 
     def on_message(headers, message):
 
-        ## print("** new message! **")
+        print("** new message! **")
 
-        ## print("message / ", message)
+        print("message / ", message)
 
-        print("JMS url -> ", message.body)
+        print("messagebody / ", message.body)
 
-        ## print("checking the value does or doesnt exist in db")
+        print("checking the value does or doesnt exist in db")
 
         alreadyExistsSuccess = RestAPIClient.searchURLSuccesses(message.body)
 
-        ## print("alreadyExistsSuccess / ", alreadyExistsSuccess)
+        print("alreadyExistsSuccess / ", alreadyExistsSuccess)
 
         if alreadyExistsSuccess.lower().strip() == "true":
     
-            ## print("this record exists (successes). returning. doing nothing")
+            print("this record exists (successes). returning. doing nothing")
 
-            ## return
+            return
 
-            print ("skipped!!")
-            
-        ## print("didn't exist in success. check in errors now.")
+        print("didn't exist in success. check in errors now.")
 
         alreadyExistsFailure = RestAPIClient.searchURLFailures(message.body)
 
-        ## print("alreadyExistsFailure / ", alreadyExistsFailure)
+        print("alreadyExistsFailure / ", alreadyExistsFailure)
 
         if alreadyExistsFailure.lower().strip() == "true":
 
-            ## print("this record exists (failures). returning. doing nothing")
+            print("this record exists (failures). returning. doing nothing")
 
-            ##return
-            
-            print ("skipped!!")
+            return
 
         print("didn't exist in success or errors. will process now")
 
@@ -97,31 +62,40 @@ class MyListener(stomp.ConnectionListener):
         RestAPIClient.addDownloadFailure(message.body, "error from onmessage (consumer.py)")
 
 print("## ")
-print(f"## starting > JMS consumer ([{jmsQueue}]")
+print("## starting > (timed) JMS consumer (yankee-sully-channels-monthly)")
 print("## ")
 
-writeSullyURLToJMS()
-connection_details = [(host, port)]
 
-conn = stomp.Connection(connection_details)
+conn = stomp.Connection([('67.220.70.106', 61613)])
 
 conn.set_listener('', MyListener())
 
-conn.connect(username, password, wait=True)
+conn.connect('admin', 'password', wait=True)
 
-conn.subscribe(f'/queue/{jmsQueue}', id=1, ack='auto', headers={'activemq.prefetchSize': 10})
+conn.subscribe('/queue/yankee-sully-channels-monthly', id=1, ack='auto', headers={'activemq.prefetchSize': 10})
 
-print(f"Subscribed to queue: {jmsQueue}")
-    
 print("## ")
-print(f"## notification > connected to JMS server [{jmsQueue}] (upgrade)")
+print("## notification > connected to JMS server (yankee-sully-channels-monthly) 3333")
 print("## ")
 
-while True:
+start_time = time.time()
+
+print("start time /",start_time)
+
+counter=0
+
+while time.time() - start_time < 60:
+
+    counter = counter + 1
 
     pass
 
+    print("waiting for 60 seconds of elapsed time: ",counter)
+
     time.sleep(1)
-       
+
+print("it's been 60 seconds. what up though?")
+    
 conn.disconnect()
 
+print("disconnected from JMS. i'll holla!")

crontabX

@@ -1,7 +1,7 @@
 SHELL=/bin/bash
 PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
-48 20 * * *  /yankee-gnome-fire-consumer/consumer.bash >> /var/log/yankee.log
+* * * * * /yankee-gnome-fire-consumer/consumer.bash >> /var/log/yankee.log
 
 * * * * * /yankee-gnome-fire-consumer/ping.bash >> /var/log/cron.log 2>&1
 

smart-cron-builder.py

@@ -42,11 +42,11 @@ def buildCronExecuteTime():
 
     current_time = datetime.datetime.now()
 
-    new_minute = (current_time.minute + 2) % 60
+    new_minute = (current_time.minute + 2) % 60  # The % operator performs the wrap-around
 
     print("new_minute / ",new_minute)
 
-    new_hour = (current_time.hour + 5) % 24
+    new_hour = current_time.hour + (current_time.minute + 5) // 60 # Integer division for hour increment
 
     print("new_hour / ",new_hour)
 
@@ -56,11 +56,11 @@ def buildCronExecuteTime():
 
 def buildCronSituation():
 
-    ##return "* * * * * /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
+    return "* * * * * /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
 
-    return buildCronExecuteTime() + " /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
+    ##return buildCronExecuteTime() + "/yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
 
-    ##return buildCronExecuteTime() +" /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName() + " 2>&1"
+    ##return buildCronExecuteTime() +"/yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName() + " 2>&1"
 
     ##return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"