From 9a787b2a580f28c30be827d12e80fce8a2170a60 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 6 Feb 2025 12:51:43 -0500 Subject: [PATCH] automated terminal push --- .dockerignore | 11 ++ Dockerfile | 17 +++ compose.bash | 32 ++++ compose.yaml | 158 ++++++++++++++++++++ docs/api/twitch/channel.md | 14 ++ docs/api/twitch/twitchdata.com/stub.md | 65 ++++++++ docs/golive.md | 40 +++++ docs/index.md | 17 +++ docs/license.md | 9 ++ docs/view.sql.md | 25 ++++ gemini | 98 ++++++++++++ nbactions.xml | 55 +++++++ pom.xml | 99 ++++++++++++ readme.md | 16 ++ test/failure/eval-failure-check-exists.bash | 49 ++++++ test/failure/eval-failure-create-new.bash | 24 +++ test/success/eval-success-check-exists.bash | 19 +++ test/success/eval-success-create-new.bash | 24 +++ 18 files changed, 772 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 compose.bash create mode 100644 compose.yaml create mode 100644 docs/api/twitch/channel.md create mode 100644 docs/api/twitch/twitchdata.com/stub.md create mode 100644 docs/golive.md create mode 100644 docs/index.md create mode 100644 docs/license.md create mode 100644 docs/view.sql.md create mode 100644 gemini create mode 100644 nbactions.xml create mode 100644 pom.xml create mode 100644 readme.md create mode 100644 test/failure/eval-failure-check-exists.bash create mode 100644 test/failure/eval-failure-create-new.bash create mode 100644 test/success/eval-success-check-exists.bash create mode 100644 test/success/eval-success-create-new.bash diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..eb1c207 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +volumes/ + +.git + +.pristine + +.trash + +.recycle + +.backup diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ed64fb3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM maven:3.8.7-openjdk-18-slim AS mavenBuild + +WORKDIR / + +COPY . . + +RUN mvn install -DskipTests + +FROM eclipse-temurin:18-jre-alpine + +COPY --from=mavenBuild /target/yankee-twitch-data-api-1.0.jar /yankee-twitch-data-api.jar + +COPY --from=mavenBuild /src/main/resources/application.properties /application.properties + +COPY --from=mavenBuild /src/main/resources/eighteen.stub.html /eighteen.stub.html + +CMD ["java", "-jar", "/yankee-twitch-data-api.jar"] diff --git a/compose.bash b/compose.bash new file mode 100644 index 0000000..c4e8597 --- /dev/null +++ b/compose.bash @@ -0,0 +1,32 @@ +#!/bin/bash + +## + +set -e + +set -x + +## + +reset + +clear + +## + +echo +echo "STARTED / yankee / twitchtracker / channel rankings" +echo + + +## + +docker compose down --remove-orphans + +#git pull + +docker compose up --build -d + +echo +echo "RUNNING / yankee / twitchtracker / channel rankings" +echo diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..07ccab3 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,158 @@ + +services: + + +########################################## +## +## aventador / network +## +########################################## + + aventador-proxy: + + container_name: aventador-proxy + + image: nginxproxy/nginx-proxy:1.6 + + restart: unless-stopped + + ports: + + - 80:80 + + - 443:443 + + volumes: + + - /var/run/docker.sock:/tmp/docker.sock:ro + + - /var/docker/nginx/html:/usr/share/nginx/html + + - /var/docker/nginx/certs:/etc/nginx/certs + + - /var/docker/nginx/vhost:/etc/nginx/vhost.d + + logging: + + options: + + max-size: "10m" + + max-file: "3" + + aventador-letsencrypt: + + container_name: aventador-letsencrypt + + image: jrcs/letsencrypt-nginx-proxy-companion + + restart: unless-stopped + + volumes_from: + + - aventador-proxy + + volumes: + + - /var/run/docker.sock:/var/run/docker.sock + + - /var/docker/nginx/acme:/etc/acme.sh + + environment: + + DEFAULT_EMAIL: embanet@gmail.com + +## +## relational database server +## +## (vanilla mariadb w/ shinobi academy training db option) +## + + yankee-database-server: + + container_name: yankee-database-server + + image: softwareshinobi/shinobi-academy-database-server + + ports: + + - "3306:3306" + + environment: + + MYSQL_ROOT_PASSWORD: aggiepride + + MYSQL_DATABASE: yankee + + MYSQL_USER: yankee + + MYSQL_PASSWORD: yankee + +## +## web assessible query tool sitting on top of the database server +## + yankee-database-query: + + container_name: yankee-database-query + + image: beeyev/phpmyadmin-lightweight + + ports: + + - "3380:80" + + depends_on: + + - yankee-database-server + + links: + + - yankee-database-server + + environment: + + PMA_HOST: yankee-database-server:3306 + + VIRTUAL_HOST: database.yankee.embanet.online + + LETSENCRYPT_HOST: database.yankee.embanet.online + +#### + +## +## central computing for all yankee operations +## + + yankee-twitch-data-api: + + container_name: yankee-twitch-data-api + + image: valorantdigital/yankee-twitch-data-api + + depends_on: + + - yankee-database-server + + restart: unless-stopped + + ports: + + - 48888:8888 + + environment: + + DB_HOST: yankee-database-server + + DB_PORT: 3306 + + DB_NAME: yankee + + DB_USER: yankee + + DB_PASS: yankee + + VIRTUAL_PORT: 8888 + + VIRTUAL_HOST: apis.yankee.embanet.online + + LETSENCRYPT_HOST: apis.yankee.embanet.online diff --git a/docs/api/twitch/channel.md b/docs/api/twitch/channel.md new file mode 100644 index 0000000..fdc7713 --- /dev/null +++ b/docs/api/twitch/channel.md @@ -0,0 +1,14 @@ +# API + +http://localhost:8888/twitch/channel/truncate + +## load the top 1000 + +go get 50x rankings, 20 times. truncates before all operations. + +localhost:8888/twitchtracker.com/channel/load/ + +## +http://localhost:8888/twitchtracker.com/channel/load/2 + +load page two of the rankings diff --git a/docs/api/twitch/twitchdata.com/stub.md b/docs/api/twitch/twitchdata.com/stub.md new file mode 100644 index 0000000..912b87c --- /dev/null +++ b/docs/api/twitch/twitchdata.com/stub.md @@ -0,0 +1,65 @@ +# TwitchTrackerStubController API Documentation + +This document outlines the functionalities provided by the `TwitchTrackerStubController` class. + +**Note:** This API provides stubbed data, meaning it simulates real data from `twitchtracker.com` but doesn't interact with the actual website. + +**Base URL:** `http://localhost:8888/twitchtracker.com/channel/stub` + +**CORS:** Enabled + +### Endpoints + +#### 1. GET /rankings + +* **Description:** This endpoint retrieves a single page (50 channels) of stubbed Twitch channel rankings data. +* **Response:** String containing HTML data representing the rankings table. + +**Example Usage:** + +``` +GET http://localhost:8888/twitchtracker.com/channel/stub/rankings +``` + +#### 2. GET /load/ + +* **Description:** This endpoint fetches and returns a list of 1000 stubbed Twitch channels with their rankings. +* **Response:** List of `TwitchChannel` objects containing channel information. + +**Example Usage:** + +``` +GET http://localhost:8888/twitchtracker.com/channel/stub/load/ +``` + +**Implementation Details:** + +* This endpoint calculates the number of pages required to retrieve 1000 channels based on the `channelsPerTwitchTrackerPage` constant (default 50). +* It then iterates through each page number and calls the `loadSingleTwitchTrackerPageByNumber` endpoint to retrieve the data. +* Finally, it combines all retrieved channels into a single list and returns it. + +#### 3. GET /load/{pageNumber} + +* **Description:** This endpoint retrieves a single page (50 channels) of stubbed Twitch channel rankings data based on the provided page number. +* **Path Variable:** `pageNumber` (integer) - Specifies the desired page number. +* **Response:** List of `TwitchChannel` objects containing channel information for the requested page. + +**Example Usage:** + +``` +GET http://localhost:8888/twitchtracker.com/channel/stub/load/2 +``` + +**Implementation Details:** + +* This endpoint fetches a single page of stubbed HTML data representing the rankings table. +* It then parses the HTML and converts it into a list of `TwitchChannel` objects using the `twitchTrackerService.convert_html_list_twitch_channels` method. +* Finally, it saves each retrieved channel to the database using the `twitchChannelService.save` method. (This behavior might be specific to your application's use case.) + + +**Additional Notes:** + +* The `sleepRandomly` method introduces a random delay between page fetches to simulate realistic network behavior. +* This documentation assumes familiarity with Spring annotations like `@GetMapping`, `@PathVariable`, and `@Autowired`. + + diff --git a/docs/golive.md b/docs/golive.md new file mode 100644 index 0000000..b67f383 --- /dev/null +++ b/docs/golive.md @@ -0,0 +1,40 @@ + +1. log in to linux server +2. clone repository + +``` +https://code.softwareshinobi.digital/yankee/twitchtracker-channel-rankings.git +`` + +3. go into directory + +4. + + +docker compose.bash + +it is assumed you already have docker installerd. + +so then run + +``` +sudo bash provision.bash +``` + +## load the fake data to spec first + +http://aventador.embanet.softwareshinobi.digital:48888/twitchtracker.com/channel/stub/load/ + +## populate the sql query to load the view + + + + +## database loging + +phpmyadmin, lets you manage shit inside the db. + +url : + +user: +pass: diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..000ea34 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,17 @@ +# Welcome to MkDocs + +For full documentation visit [mkdocs.org](https://www.mkdocs.org). + +## Commands + +* `mkdocs new [dir-name]` - Create a new project. +* `mkdocs serve` - Start the live-reloading docs server. +* `mkdocs build` - Build the documentation site. +* `mkdocs -h` - Print help message and exit. + +## Project layout + + mkdocs.yml # The configuration file. + docs/ + index.md # The documentation homepage. + ... # Other markdown pages, images and other files. diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 0000000..2071b23 --- /dev/null +++ b/docs/license.md @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/docs/view.sql.md b/docs/view.sql.md new file mode 100644 index 0000000..fbd76d6 --- /dev/null +++ b/docs/view.sql.md @@ -0,0 +1,25 @@ +# the view for custom reporting + +## Create the View + +``` + +## Drop the view + +DROP VIEW IF EXISTS report_twitchdata_top_1500_channels; + +CREATE VIEW report_twitchdata_top_1500_channels AS +SELECT + `tag` "username",`url`,`logo` "logo_url", `all_time_peak_viewers`, `avg_viewers` "average_viewers", `followers_gained`, `hours_streamed`, `hours_watched`, `rank`, `total_followers`, +CASE + WHEN total_views = -1 THEN 'null' + ELSE total_views + END AS total_views +from twitch_channel; + +-- Select data from the view + +SELECT * FROM report_twitchdata_top_1500_channels; + + +``` diff --git a/gemini b/gemini new file mode 100644 index 0000000..3b79f24 --- /dev/null +++ b/gemini @@ -0,0 +1,98 @@ +package com.example.filedownload.model; + +import jakarta.persistence.*; +import java.time.LocalDateTime; + +@Entity +@Table(name = "file_downloads") +public class FileDownload { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false) + private String url; + + @Column(nullable = false) + private String hostIp; + + @Column(nullable = false) + private LocalDateTime dateTime; + + // Constructors, getters and setters +} + + + + +package com.example.filedownload.service; + +import com.example.filedownload.model.FileDownload; +import com.example.filedownload.repository.FileDownloadRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; + +@Service +public class FileDownloadService { + + @Autowired + private FileDownloadRepository fileDownloadRepository; + + public FileDownload save(String url, String hostIp) { + FileDownload fileDownload = new FileDownload(); + fileDownload.setUrl(url); + fileDownload.setHostIp(hostIp); + fileDownload.setDateTime(LocalDateTime.now()); + return fileDownloadRepository.save(fileDownload); + } +} + +package com.example.filedownload.service; + +import com.example.filedownload.model.FileDownload; +import com.example.filedownload.repository.FileDownloadRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; + +@Service +public class FileDownloadService { + + @Autowired + private FileDownloadRepository fileDownloadRepository; + + public FileDownload save(String url, String hostIp) { + FileDownload fileDownload = new FileDownload(); + fileDownload.setUrl(url); + fileDownload.setHostIp(hostIp); + fileDownload.setDateTime(LocalDateTime.now()); + return fileDownloadRepository.save(fileDownload); + } +} + + +package com.example.filedownload.controller; + +import com.example.filedownload.model.FileDownload; +import com.example.filedownload.service.FileDownloadService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class FileDownloadController { + + @Autowired + private FileDownloadService fileDownloadService; + + @GetMapping("/doesexist/{url}") + public boolean doesExist(@PathVariable String url) { + String fullUrl = "google.com/" + url; // Assuming "google.com" is the base URL + return fileDownloadService.existsByUrl(fullUrl); + } +} diff --git a/nbactions.xml b/nbactions.xml new file mode 100644 index 0000000..c2fc1f6 --- /dev/null +++ b/nbactions.xml @@ -0,0 +1,55 @@ + + + + run + + jar + + + process-classes + org.codehaus.mojo:exec-maven-plugin:3.1.0:exec + + + -DDB_HOST=aventador.embanet.online -DDB_PORT=3306 -DDB_NAME=aventador -DDB_USER=aventador -DDB_PASS=aventador + ${exec.vmArgs} -classpath %classpath ${exec.mainClass} ${exec.appArgs} + + edu.gsyoung.twitch.TwitchDataAcquisition + java + + + + debug + + jar + + + process-classes + org.codehaus.mojo:exec-maven-plugin:3.1.0:exec + + + -DDB_HOST=aventador.embanet.online -DDB_PORT=3306 -DDB_NAME=aventador -DDB_USER=aventador -DDB_PASS=aventador -agentlib:jdwp=transport=dt_socket,server=n,address=${jpda.address} + ${exec.vmArgs} -classpath %classpath ${exec.mainClass} ${exec.appArgs} + + edu.gsyoung.twitch.TwitchDataAcquisition + java + true + + + + profile + + jar + + + process-classes + org.codehaus.mojo:exec-maven-plugin:3.1.0:exec + + + -DDB_HOST=aventador.embanet.online -DDB_PORT=3306 -DDB_NAME=aventador -DDB_USER=aventador -DDB_PASS=aventador + ${exec.vmArgs} -classpath %classpath ${exec.mainClass} ${exec.appArgs} + edu.gsyoung.twitch.TwitchDataAcquisition + java + + + + diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..7a73ada --- /dev/null +++ b/pom.xml @@ -0,0 +1,99 @@ + + + + 4.0.0 + + + + org.springframework.boot + + spring-boot-starter-parent + + 3.1.0 + + + + + + edu.gsyoung + + yankee-twitch-data-api + + yankee-twitch-data-api + + yankee-twitch-data-api + + 1.0 + + + + 17 + + + + + + + org.jsoup + jsoup + 1.13.1 + jar + + + + org.springframework.boot + spring-boot-starter-actuator + + + + org.springframework.boot + spring-boot-starter-data-jpa + + + + org.springframework.boot + spring-boot-starter-thymeleaf + + + + org.springframework.boot + spring-boot-starter-validation + + + + org.springframework.boot + spring-boot-starter-web + + + + org.projectlombok + lombok + true + + + + org.mariadb.jdbc + mariadb-java-client + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + + + + + + diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..0eccaf7 --- /dev/null +++ b/readme.md @@ -0,0 +1,16 @@ +# twitch saas! + +## sit rep + +[DONE] the monthly stats data is loading but the repositories are all broken. + +[NOW] now we need to enrich the html so that the js and the css load and properly render the page and render the table data + + +## later + +fix all the repos + +create the click page for the phase one activities + +add a menu to the click page diff --git a/test/failure/eval-failure-check-exists.bash b/test/failure/eval-failure-check-exists.bash new file mode 100644 index 0000000..cd10234 --- /dev/null +++ b/test/failure/eval-failure-check-exists.bash @@ -0,0 +1,49 @@ +reset + +clear + +set -e + +#set -x + +## + +echo +echo "##" +echo "##" +echo "##" +echo + +## + +curl -X GET http://localhost:8888/channel/monthly/games/failure/ + +sleep 4 + +echo +echo "##" +echo "##" +echo "##" +echo + +curl -X GET \ +http://localhost:8888/channel/monthly/games/failure/exists \ +-H 'Content-Type: application/json' \ +-d '{ +"url": "https://links.embanet.com/talisman.txt" +}' + +sleep 4 + +echo +echo "##" +echo "##" +echo "##" +echo + +curl -X GET \ +http://localhost:8888/channel/monthly/games/failure/exists \ +-H 'Content-Type: application/json' \ +-d '{ +"url": "https://www.garudanet.com/talisman.txt" +}' diff --git a/test/failure/eval-failure-create-new.bash b/test/failure/eval-failure-create-new.bash new file mode 100644 index 0000000..59517fb --- /dev/null +++ b/test/failure/eval-failure-create-new.bash @@ -0,0 +1,24 @@ +#!/bin/bash + +## + +reset + +clear + +## + +set -e + +set -x + +## + +echo "##" +echo "## creating new [failure] record in the database" +echo "##" + +curl -X POST \ +http://localhost:8888/channel/monthly/games/failure/add \ +-H 'Content-Type: application/json' \ +-d '{"url": "https://www.garudanet.com/talisman.txt","hostIP":"falcon"}' diff --git a/test/success/eval-success-check-exists.bash b/test/success/eval-success-check-exists.bash new file mode 100644 index 0000000..866dd0b --- /dev/null +++ b/test/success/eval-success-check-exists.bash @@ -0,0 +1,19 @@ +echo "##" +echo "##" +echo "##" + +curl -X GET https://osiris.yankee.valorantdigital.com/channel/monthly/games/success/ + +curl -X GET \ +https://osiris.yankee.valorantdigital.com/channel/monthly/games/success/exists \ +-H 'Content-Type: application/json' \ +-d '{ +"url": "https://www.embanet.com/talisman.txt" +}' + +curl -X GET \ +https://osiris.yankee.valorantdigital.com/channel/monthly/games/success/exists \ +-H 'Content-Type: application/json' \ +-d '{ +"url": "https://www.embanet.com/talisman.md" +}' diff --git a/test/success/eval-success-create-new.bash b/test/success/eval-success-create-new.bash new file mode 100644 index 0000000..9a27136 --- /dev/null +++ b/test/success/eval-success-create-new.bash @@ -0,0 +1,24 @@ +#!/bin/bash + +## + +reset + +clear + +## + +set -e + +set -x + +## + +echo "##" +echo "## creating new [success] record in the database" +echo "##" + +curl -X POST \ +https://osiris.yankee.valorantdigital.com/channel/monthly/games/success/add \ +-H 'Content-Type: application/json' \ +-d '{"url": "https://www.embanet.com/talisman.txt","hostIP":"asdf"}'