31 Commits

Author SHA1 Message Date
Your Name
39b2608a36 serious progress 2025-03-12 09:16:22 -04:00
Your Name
782e6fc655 more 2025-03-11 21:20:17 -04:00
Software Shinobi
4ec5f446e8 automated terminal push 2025-03-11 20:44:04 -04:00
Software Shinobi
dc51e2e646 automated terminal push 2025-03-11 20:42:48 -04:00
Software Shinobi
c671cb5d61 automated terminal push 2025-03-11 20:40:40 -04:00
Software Shinobi
73cb40154c automated terminal push 2025-03-11 20:32:29 -04:00
Your Name
19161b872c add sleeps back 2025-03-10 13:52:44 -04:00
Your Name
a2d7eb6896 send fail on file not found after 5 checks 2025-03-10 13:49:50 -04:00
Your Name
118e784e4b update file system and jsm references 2025-03-10 11:13:21 -04:00
Your Name
07981fa5ce automated terminal push 2025-02-13 15:57:57 -05:00
Your Name
20358fffed automated terminal push 2025-02-13 15:47:55 -05:00
Your Name
0bbc89d9be automated terminal push 2025-02-13 15:46:49 -05:00
Your Name
7a38b01406 automated terminal push 2025-02-13 15:45:05 -05:00
Your Name
92577c120f automated terminal push 2025-02-13 08:46:45 -05:00
Your Name
f9605dbce3 automated terminal push 2025-02-13 08:35:22 -05:00
Your Name
962f35d2e9 automated terminal push 2025-02-13 08:09:04 -05:00
Your Name
70be610e92 automated terminal push 2025-02-13 07:17:06 -05:00
Your Name
f2c499cebc automated terminal push 2025-02-13 07:01:47 -05:00
Your Name
4a5b66ec21 automated terminal push 2025-02-13 06:58:55 -05:00
Your Name
2686c62bda automated terminal push 2025-02-13 06:53:51 -05:00
Your Name
4272418cb0 automated terminal push 2025-02-13 06:52:00 -05:00
Your Name
e6557e2820 automated terminal push 2025-02-13 06:45:36 -05:00
Your Name
dbd1d53cb4 automated terminal push 2025-02-13 06:38:04 -05:00
Your Name
3a557ba120 automated terminal push 2025-02-13 06:33:56 -05:00
Your Name
367a5e075b automated terminal push 2025-02-13 06:31:06 -05:00
Your Name
2dbc856a8b automated terminal push 2025-02-12 15:08:14 -05:00
Your Name
87f1536e95 automated terminal push 2025-02-12 15:07:33 -05:00
Your Name
eb71aa4c48 automated terminal push 2025-02-12 14:58:39 -05:00
Your Name
6667c774c3 automated terminal push 2025-02-12 11:06:22 -05:00
2b86141b99 Merge pull request 'automated terminal push' (#1) from feature/hostname into production
Reviewed-on: https://code.yankee.embanet.online/yankee/yankee-gnome-fire-consumer/pulls/1
2025-02-10 15:16:12 +00:00
Your Name
9d6d996338 automated terminal push 2025-02-10 09:39:55 -05:00
50 changed files with 568 additions and 123 deletions

View File

@@ -3,25 +3,54 @@ import time
import re
import fnmatch
def dowork():
def check_download_exists_matching_url_version_one(url):
################################################################################################
print ("enter > downloads > exist check > version 1")
url = "https://sullygnome.com/channel/kaicenat/2023january/streams"
match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
print("checking disk for the stuff...")
if not match:
print("url", url)
print(f"Invalid URL format. Could not extract channel name, year, or month.")
fileExists = check_download_exists_matching_url(url)
return
print("file already exists / ",fileExists)
channel_name, year, month = match.groups()
################################################################################################
print("channel / ",channel_name)
def check_download_exists_matching_url(url):
#print("year / ",year)
print ("hello?")
#print("month / ",month)
#file_pattern = f"{channel_name}*{month}*{year}*.csv"
## this is a problem. it works but if there 2 channels.
## redbull and redbull2 this is throw false positives. ugh
file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
for filename in os.listdir("/home/yankee/Downloads/"):
updated = filename.lower()
#print("file in downloads / ",updated)
if fnmatch.fnmatch(updated, file_pattern):
print(f"Found matching file: {updated}")
return True
else:
print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'. (version one)")
return False
def check_download_exists_matching_url_version_two(url):
print ("enter > check_download_exists_matching_url_version_two")
match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
@@ -39,31 +68,69 @@ def check_download_exists_matching_url(url):
print("month / ",month)
#file_pattern = f"{channel_name}*{month}*{year}*.csv"
## this is a problem. it works but if there 2 channels.
## redbull and redbull2 this is throw false positives. ugh
file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
for filename in os.listdir("/root/Downloads/"):
for filename in os.listdir("/home/yankee/Downloads/"):
updated = filename.lower()
print("file in downloads / ",updated)
print("found file (v2) / ",updated)
if fnmatch.fnmatch(updated, file_pattern):
##
print(f"Found matching file: {updated}")
hasKeyDateParts = does_filename_contain_key_date_parts(updated, month, year)
return True
print("\thasKeyDateParts / ",hasKeyDateParts)
else:
##
print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'.")
hasTargetChannelName = does_filename_contain_target_channel_name(updated,channel_name)
print("\thasTargetChannelName / ",hasTargetChannelName)
##
print ("exit < check_download_exists_matching_url_version_two")
def does_filename_contain_target_channel_name(url,channel):
regex = rf"\({re.escape(channel)}\)"
match = re.search(regex, url)
results = bool(match)
print("results / ",results)
match = re.search("\({channel}\)", url)
if match:
print("yup!")
extracted_text = match.group(1).lower()
print("extracted: /",extracted_text)
return False
def shutDownRobot():
## return extracted_text == target_string.lower()
else:
print("nope!")
return False
def does_filename_contain_key_date_parts(filename, monthString, yearNumber):
print("enter -> does_filename_contain_key_date_parts()")
regex = rf"{re.escape(monthString)} {yearNumber}"
doesContain = bool(re.search(regex, filename, re.IGNORECASE))
print("doesContain / ",doesContain)
return doesContain
print("shutting down the robot now")

View File

@@ -1,6 +1,7 @@
import requests
import json
import socket
import netifaces
## https://www.nylas.com/blog/use-python-requests-module-rest-apis/#how-to-use-python-requests
## https://www.abstractapi.com/guides/ip-geolocation/get-ip-address-python
@@ -21,13 +22,25 @@ def ip():
return None
def hostname():
try:
return socket.gethostname()
except Exception as e:
print(f"Error getting hostname: {e}")
return "unknown_host"
def addDownloadSuccess(url):
print("enter -> addDownloadSuccess()")
print("param / url / ",url)
data = {"url": url, "hostIP": ip(),"ip": ip(),}
data = {"url": url, "hostIP": ip(),"ip": ip(),"hostName":hostname()}
json_data = json.dumps(data)
@@ -37,20 +50,20 @@ def addDownloadSuccess(url):
headers = {"Content-Type": "application/json"}
response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/success/add", data=json_data, headers=headers)
print(response.text)
def addDownloadFailure(url):
def addDownloadFailure(url, error):
print("enter -> addDownloadFailure()")
print("param / url / ",url)
print("param / error / ",error)
data = {"url": url, "hostIP": ip(),"ip": ip(),}
data = {"url": url, "hostIP": ip(),"ip": ip(),"hostName":hostname(), "reason":error}
json_data = json.dumps(data)
@@ -60,50 +73,16 @@ def addDownloadFailure(url):
headers = {"Content-Type": "application/json"}
response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/failure/add", data=json_data, headers=headers)
print(response.text)
#print("params", params)
#response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/success/exists", headers=headers, data=params, params=params)
#print("response2", response.json())
def searchURLSuccesses(url):
print("enter -> callGET()")
## print("enter -> callGET()")
print("param / url / ",url)
## print("param / url / ",url)
params = dict(
origin='Chicago,IL',
destination='Los+Angeles,CA',
waypoints='Joplin,MO|Oklahoma+City,OK',
sensor='false',
url2=url
)
#data = {"name": "Jane Smith", "email": "janesmith@example.com", "url": url }
data = { "url": url }
headers = {"Content-Type": "application/json"}
@@ -112,27 +91,16 @@ def searchURLSuccesses(url):
response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/success/exists", data=json_data, headers=headers)
print(response.text)
## print(response.text)
return response.text
def searchURLFailures(url):
print("enter -> callGET()")
## print("enter -> callGET()")
print("param / url / ",url)
## print("param / url / ",url)
params = dict(
origin='Chicago,IL',
destination='Los+Angeles,CA',
waypoints='Joplin,MO|Oklahoma+City,OK',
sensor='false',
url2=url
)
#data = {"name": "Jane Smith", "email": "janesmith@example.com", "url": url }
data = { "url": url }
headers = {"Content-Type": "application/json"}
@@ -141,6 +109,6 @@ def searchURLFailures(url):
response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/failure/exists", data=json_data, headers=headers)
print(response.text)
## print(response.text)
return response.text

View File

@@ -10,14 +10,12 @@ import RestAPIClient
import DiskUtils
from selenium.webdriver import FirefoxOptions
## https://sullygnome.com/channel/kaicenat/365/games"
def download(url):
print(f"launch -> SullyGnomeRobot -> download() -> {url}")
opts = FirefoxOptions()
opts.add_argument("--headless")
driver=webdriver.Firefox(options=opts)
@@ -28,7 +26,6 @@ def download(url):
print("url / ", driver.current_url)
##########################################
print("stage / start / tblControl_length check")
@@ -49,7 +46,7 @@ def download(url):
print(f"Element with class name 'tblControl_length' not found: {e}")
RestAPIClient.addDownloadFailure(url)
RestAPIClient.addDownloadFailure(url, "Element with class name 'tblControl_length' not found")
driver.quit()
@@ -75,7 +72,7 @@ def download(url):
print(f"Element with class name 'TableExportLinkButton' not found: {e}")
RestAPIClient.addDownloadFailure(url)
RestAPIClient.addDownloadFailure(url,"Element with class name 'TableExportLinkButton' not found")
driver.quit()
@@ -87,17 +84,47 @@ def download(url):
print("file download started. now checking if exist before closing")
counter = 0
max_iterations = 5
while True:
print(f"Loop iteration: {counter}")
counter += 1
if counter == max_iterations:
print(f"Loop exited after {max_iterations} iterations.")
print("shutting down the robot")
driver.quit()
print("robot shut down. breaking. and dying.")
RestAPIClient.addDownloadFailure(url,"file not found on disk")
print("stage / complete / database updated w/ FAILURE")
break
print("checking is file exists... [loop]")
existsAlready = DiskUtils.check_download_exists_matching_url(url)
existsAlreadyVersionOne = DiskUtils.check_download_exists_matching_url_version_one(url)
print("file already exists / ",existsAlready)
print("file already exists (v1) / ",existsAlreadyVersionOne)
if existsAlready:
existsAlreadyVersionTwo = DiskUtils.check_download_exists_matching_url_version_two(url)
print(f"The file matching the url '{url}' exists.")
print("file already exists (v2) / ",existsAlreadyVersionTwo)
time.sleep(40)
if existsAlreadyVersionOne:
print(f"The file matching the url '{url}' DOES INDEED exists. (version 1)")
print("sleeping for 4 secs before shutting down robot")
@@ -109,6 +136,10 @@ def download(url):
print("robot shut down. breaking. and dying.")
RestAPIClient.addDownloadSuccess(url)
print("stage / complete / database updated w/ success")
break
else:
@@ -123,8 +154,4 @@ def download(url):
##########################################
RestAPIClient.addDownloadSuccess(url)
print("stage / complete / database updated w/ success")
print(f"complete -> SullyGnomeRobot -> download() -> {url}")

View File

@@ -3,15 +3,13 @@
set -e
set -x
##
echo "what up????"
##cd /yankee-gnome-fire-consumer
cd /yankee-gnome-fire-consumer
echo "##"
echo "## init > yankee > consumer"
echo "##"
pwd
ls
whoami
python3 consumer.py

View File

@@ -3,43 +3,80 @@ import time
import SullyGnomeRobot
import RestAPIClient
### start variables ###
jmsQueue="asian-tester"
host = '38.107.226.34'
port = 61613
username = 'admin'
password = 'password'
### end variables ###
print("## ")
print("## starting > JMS consumer (yankee-sully-channels-monthly)")
print(f"## starting > JMS consumer app!")
print("## ")
def writeSullyURLToJMS():
channel="https://sullygnome.com/channel/yaritaiji/2024october/games"
print(f"launch -> jms writer -> channel-> {channel}")
conn = stomp.Connection([('osiris.yankee.embanet.online', 61613)])
conn.connect('admin', 'password', wait=True)
try:
conn.send(body=channel.strip(), destination=f'/queue/{jmsQueue}')
finally:
conn.disconnect()
class MyListener(stomp.ConnectionListener):
def on_message(headers, message):
print("** new message! **")
## print("** new message! **")
print("message / ", message)
## print("message / ", message)
print("messagebody / ", message.body)
print("JMS url -> ", message.body)
print("checking the value does or doesnt exist in db")
## print("checking the value does or doesnt exist in db")
alreadyExistsSuccess = RestAPIClient.searchURLSuccesses(message.body)
print("alreadyExistsSuccess / ", alreadyExistsSuccess)
## print("alreadyExistsSuccess / ", alreadyExistsSuccess)
if alreadyExistsSuccess.lower().strip() == "true":
print("this record exists (successes). returning. doing nothing")
## print("this record exists (successes). returning. doing nothing")
return
## return
print("didn't exist in success. check in errors now.")
print ("skipped!!")
## print("didn't exist in success. check in errors now.")
alreadyExistsFailure = RestAPIClient.searchURLFailures(message.body)
print("alreadyExistsFailure / ", alreadyExistsFailure)
## print("alreadyExistsFailure / ", alreadyExistsFailure)
if alreadyExistsFailure.lower().strip() == "true":
print("this record exists (failures). returning. doing nothing")
## print("this record exists (failures). returning. doing nothing")
return
##return
print ("skipped!!")
print("didn't exist in success or errors. will process now")
@@ -51,28 +88,33 @@ class MyListener(stomp.ConnectionListener):
print(f"exception / something went wrong when the sully gnome robot was downloading: {e}")
RestAPIClient.addDownloadFailure(message.body)
RestAPIClient.addDownloadFailure(message.body, e)
def on_error(headers, message):
print('received an error "%s"' % message)
RestAPIClient.addDownloadFailure(message.body)
RestAPIClient.addDownloadFailure(message.body, "error from onmessage (consumer.py)")
print("## ")
print("## starting > JMS consumer (yankee-sully-channels-monthly)")
print(f"## starting > JMS consumer ([{jmsQueue}]")
print("## ")
conn = stomp.Connection([('67.220.70.106', 61613)])
writeSullyURLToJMS()
connection_details = [(host, port)]
conn = stomp.Connection(connection_details)
conn.set_listener('', MyListener())
conn.connect('admin', 'password', wait=True)
conn.connect(username, password, wait=True)
conn.subscribe('/queue/yankee-sully-channels-monthly', id=1, ack='auto', headers={'activemq.prefetchSize': 10})
conn.subscribe(f'/queue/{jmsQueue}', id=1, ack='auto', headers={'activemq.prefetchSize': 10})
print(f"Subscribed to queue: {jmsQueue}")
print("## ")
print("## notification > connected to JMS server (yankee-sully-channels-monthly) 3333")
print(f"## notification > connected to JMS server [{jmsQueue}] (upgrade)")
print("## ")
while True:
@@ -80,5 +122,6 @@ while True:
pass
time.sleep(1)
conn.disconnect()

6
recycle/crontabX Normal file
View File

@@ -0,0 +1,6 @@
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
6 11 * * * python3 /yankee-gnome-fire-consumer/consumer.py >> /tmp/root/Downloads/.logs/yankee-node-96.255.165.82-veneno.log 2>&1

7
recycle/crontabX2 Normal file
View File

@@ -0,0 +1,7 @@
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
48 20 * * * /yankee-gnome-fire-consumer/consumer.bash >> /var/log/yankee.log
* * * * * /yankee-gnome-fire-consumer/ping.bash >> /var/log/cron.log 2>&1

4
recycle/ping.bash Executable file
View File

@@ -0,0 +1,4 @@
touch /tmp/`date +%Y_%m_%d_%H_%M_%S`.dat
ls -lha /tmp/*dat

View File

View File

@@ -0,0 +1,151 @@
import datetime
import subprocess
import socket
import os
import requests
def buildLogDirectory():
return "/root/Downloads/.logs/"
def buildLogFileName():
logFileDir = buildLogDirectory()
try:
response = requests.get("https://api.ipify.org?format=json") # Or another similar service
response.raise_for_status() # Check for HTTP errors (4xx or 5xx)
ip_data = response.json()
server_ip = ip_data['ip']
print("server_ip / ",ip_data['ip'])
server_hostname = socket.gethostname()
print("server_hostname / ",server_hostname)
node_id= logFileDir + "yankee-node-" + server_ip + "-" + server_hostname + ".log"
return node_id
except socket.gaierror:
return "!!error!!"
def buildCronExecuteTime():
current_time = datetime.datetime.now()
new_minute = (current_time.minute + 2) % 60 # The % operator performs the wrap-around
print("new_minute / ",new_minute)
new_hour = current_time.hour + (current_time.minute + 5) // 60 # Integer division for hour increment
print("new_hour / ",new_hour)
cron = str(new_minute) + " " + str(new_hour) + " " + "* * *"
return cron
def buildCronSituation():
return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"
####
cronTiming = buildCronExecuteTime()
print("cronTiming / ",cronTiming)
##
logger = buildLogFileName();
print("logger / ",logger)
##
cronFile = buildCronSituation();
print()
print(cronFile)
print()
def createLogDir():
log_dir = buildLogDirectory()
print("creating log dir",log_dir)
try:
if not os.path.exists(log_dir): # Check if the directory exists
os.makedirs(log_dir) # Create the directory (and any necessary parent directories)
print(f"Directory '{log_dir}' created successfully.")
else:
print(f"Directory '{log_dir}' already exists.")
return True # Directory exists or was created successfully
except OSError as e:
print(f"Error creating directory '{log_dir}': {e}")
return False
##
def writeCrontabSituation():
file_path ="crontabX"
crontab_text = """
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
* * * * * python3 /yankee-gnome-fire-consumer/consumer.py >> /var/log/yankee-downloader.log 2>&1
* * * * * /tmp/ping2.bash >> /var/log/cron.log 2>&1"""
try:
# Check if the file exists and delete if it does
if os.path.exists(file_path):
os.remove(file_path)
with open(file_path, "w") as f:
sep="\n"
f.write("SHELL=/bin/bash")
f.write(sep)
f.write("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
f.write(sep)
f.write(sep)
f.write(buildCronSituation())
f.write(sep)
f.write(sep)
f.write("* * * * * /tmp/ping.bash >> /var/log/cron.log 2>&1")
f.write(sep)
f.write(sep)
print(f"Crontab written to '{file_path}' successfully.")
return True
except OSError as e:
print(f"Error writing to file '{file_path}': {e}")
return False
####################
createLogDir()
writeCrontabSituation()

17
recycle/smart-cron-builder.bash Executable file
View File

@@ -0,0 +1,17 @@
reset
clear
python3 smart-cron-builder.py
####
cp crontabX /etc/cron.d/cronfile
# Give execution rights on the cron job
chmod 0644 /etc/cron.d/cronfile
# Apply the cron job
crontab -u root /etc/cron.d/cronfile
crontab -l

157
recycle/smart-cron-builder.py Executable file
View File

@@ -0,0 +1,157 @@
import datetime
import subprocess
import socket
import os
import requests
def buildLogDirectory():
return "/root/Downloads/.logs/"
def buildLogFileName():
logFileDir = buildLogDirectory()
try:
response = requests.get("https://api.ipify.org?format=json") # Or another similar service
response.raise_for_status() # Check for HTTP errors (4xx or 5xx)
ip_data = response.json()
server_ip = ip_data['ip']
print("server_ip / ",ip_data['ip'])
server_hostname = socket.gethostname()
print("server_hostname / ",server_hostname)
#node_id=logFileDir + "yankee-node-" + server_ip + "-" + server_hostname + ".log"
node_id= "/var/log/yankee.log"
return node_id
except socket.gaierror:
return "!!error!!"
def buildCronExecuteTime():
current_time = datetime.datetime.now()
new_minute = (current_time.minute + 2) % 60
print("new_minute / ",new_minute)
new_hour = (current_time.hour + 5) % 24
print("new_hour / ",new_hour)
cron = str(new_minute) + " " + str(new_hour) + " " + "* * * "
return cron
def buildCronSituation():
##return "* * * * * /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
return buildCronExecuteTime() + " /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
##return buildCronExecuteTime() +" /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName() + " 2>&1"
##return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"
####
cronTiming = buildCronExecuteTime()
print("cronTiming / ",cronTiming)
##
logger = buildLogFileName();
print("logger / ",logger)
##
cronFile = buildCronSituation();
print()
print(cronFile)
print()
def createLogDir():
log_dir = buildLogDirectory()
print("creating log dir",log_dir)
try:
if not os.path.exists(log_dir): # Check if the directory exists
os.makedirs(log_dir) # Create the directory (and any necessary parent directories)
print(f"Directory '{log_dir}' created successfully.")
else:
print(f"Directory '{log_dir}' already exists.")
return True # Directory exists or was created successfully
except OSError as e:
print(f"Error creating directory '{log_dir}': {e}")
return False
##
def writeCrontabSituation():
file_path ="crontabX"
try:
if os.path.exists(file_path):
os.remove(file_path)
with open(file_path, "w") as f:
sep="\n"
f.write("SHELL=/bin/bash")
f.write(sep)
f.write("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
f.write(sep)
f.write(sep)
f.write(buildCronSituation())
f.write(sep)
f.write(sep)
f.write("* * * * * /yankee-gnome-fire-consumer/ping.bash >> /var/log/cron.log 2>&1")
f.write(sep)
f.write(sep)
#worksf.write("* * * * * /yankee-gnome-fire-consumer/consumer.bash >> /var/log/consumer.log 2>&1")
#f.write(sep)
#f.write(sep)
print(f"Crontab written to '{file_path}' successfully.")
return True
except OSError as e:
print(f"Error writing to file '{file_path}': {e}")
return False
####################
#createLogDir()
writeCrontabSituation()