Add support for APIs (in addition to scraping)

2026-08-28 04:46:54 -04:00 · 2026-01-04 15:36:56 -05:00
parent 8a63e50228
commit 8672aadcb8
6 changed files with 119 additions and 66 deletions
@@ -1,50 +0,0 @@
-package bthtml
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"io"
-	"net/http"
-)
-
-type payloadT struct {
-	Cmd                    string  `json:"cmd"`
-	URL                    string  `json:"url"`
-	MaxTimeoutMilliseconds int     `json:"maxTimeout"`
-	WaitInSeconds          float32 `json:"waitInSeconds"`
-}
-
-func GetFromURL(flareSolverrURL, webPageURL string, loadSeconds float32) (string, error) {
-	client := &http.Client{}
-
-	payload := payloadT{
-		Cmd:                    "request.get",
-		URL:                    webPageURL,
-		MaxTimeoutMilliseconds: 60000,
-		WaitInSeconds:          loadSeconds,
-	}
-	payloadBytes, err := json.Marshal(payload)
-	if err != nil {
-		return "", errors.New("unable to marshal FlareSolverr payload to JSON: " + err.Error())
-	}
-
-	req, _ := http.NewRequest(
-		"POST",
-		flareSolverrURL,
-		bytes.NewBuffer(payloadBytes),
-	)
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", errors.New("unable to make FlareSolverr request to " + flareSolverrURL + ": " + err.Error())
-	}
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", errors.New("unable to read FlareSolverr response body: " + err.Error())
-	}
-	return string(body), nil
-}
@@ -0,0 +1,33 @@
+<input>
+    <apiMethodEndpointPayload
+    ><![CDATA[POST@https://api.enchor.us/search@{"search":"*","per_page":10,"page":1,"instrument":null,"difficulty":null,"drumType":null,"drumsReviewed":false,"sort":null,"source":"bridge"}]]></apiMethodEndpointPayload>
+    <regex
+    ><![CDATA[{\"ordering\":.*?,\"name\":\"(.*?)\",\"artist\":\"(.*?)\",\"album\":\"(.*?)\",.*?\",\"md5\":\"(.*?)\",.*?,\"charter\":\"(.*?)\",]]></regex>
+    <loadSeconds>1</loadSeconds>
+    <maxFeedItems>10</maxFeedItems>
+    <feed>
+        <title>
+            <parent>Chorus Encore</parent>
+                <captureGroupConnector />
+                <captureGroupIndex>1</captureGroupIndex>
+                <captureGroupConnector> by </captureGroupConnector>
+                <captureGroupIndex>2</captureGroupIndex>
+        </title>
+        <link><parent>https://www.enchor.us/</parent>
+            <captureGroupConnector>https://www.enchor.us/chart/</captureGroupConnector>
+            <captureGroupIndex>4</captureGroupIndex>
+        </link>
+        <description>
+            <parent>Latest YARG/CH charts published to Chorus Encore</parent>
+            <captureGroupConnector>Charter: </captureGroupConnector>
+            <captureGroupIndex>5</captureGroupIndex>
+            <captureGroupConnector> | Album: </captureGroupConnector>
+            <captureGroupIndex>3</captureGroupIndex>
+        </description>
+        <authorName>
+            <parent>Gamers</parent>
+            <captureGroupIndex>5</captureGroupIndex>
+        </authorName>
+        <authorEmail />
+    </feed>
+</input>
@@ -1,5 +1,5 @@
 <input>
-    <flareSolverrURL>http://PLACEHOLDER:8191/v1</flareSolverrURL>
+    <flareSolverrEndpoint>http://PLACEHOLDER:8191/v1</flareSolverrEndpoint>
    <regex
    ><![CDATA[<ce-search-result class=\\"card rounded-none bg-base-100 shadow-xl\\"><!----><!----><div class=\\"flex\\"><img alt=\\"Album art\\" loading=\\"lazy\\" class=\\"object-cover w-40\\" src=\\".*?\\"><!----><div class=\\"flex-grow p-3\\"><div class=\\"text-xl \[text-wrap:balance\] sm:text-2xl\\">(.*?)<\/div><!----><div><span class=\\"text-sm font-bold \[text-wrap:balance\] sm:text-base\\">Album:<\/span> (.*?)<\/div><div><span class=\\"text-sm font-bold \[text-wrap:balance\] sm:text-base\\">Genre:<\/span> .*?Charter:<\/div><a target=\\"_blank\\" class=\\"link-hover link\\" href=\\".*?\\">(.*?)<\/a><\/div><!----><!----><!----><!---->.*?for=\\"downloadVideos_(.*?)\\"]]></regex>
    <loadSeconds>1</loadSeconds>
@@ -5,10 +5,11 @@ import (
 	"encoding/xml"
 	"fmt"
 	"os"
+	"strings"

 	"github.com/gorilla/feeds"
 	"github.com/rwinkhart/backtone/atom"
-	"github.com/rwinkhart/backtone/bthtml"
+	"github.com/rwinkhart/backtone/websrc"
 	"github.com/rwinkhart/go-boilerplate/other"
 )

@@ -32,11 +33,12 @@ type feedT struct {
 	AuthorEmail feedFieldInfoT `xml:"authorEmail"`
 }
 type inputT struct {
-	FlareSolverrURL string  `xml:"flareSolverrURL"`
-	Regex           string  `xml:"regex"`
-	LoadSeconds     float32 `xml:"loadSeconds"`
-	MaxFeedItems    int     `xml:"maxFeedItems"`
-	Feed            feedT   `xml:"feed"`
+	FlareSolverrEndpoint     string  `xml:"flareSolverrEndpoint,omitempty"`
+	APIMethodEndpointPayload string  `xml:"apiMethodEndpointPayload,omitempty"`
+	Regex                    string  `xml:"regex"`
+	LoadSeconds              float32 `xml:"loadSeconds"`
+	MaxFeedItems             int     `xml:"maxFeedItems"`
+	Feed                     feedT   `xml:"feed"`
 }

 func main() {
@@ -44,10 +46,10 @@ func main() {
 	if len(os.Args) < 2 {
 		demoInputBytes, _ := xml.MarshalIndent(
 			inputT{
-				FlareSolverrURL: "http://127.0.0.1:8191",
-				Regex:           "PLACEHOLDER",
-				LoadSeconds:     2.5,
-				MaxFeedItems:    10,
+				FlareSolverrEndpoint: "http://127.0.0.1:8191",
+				Regex:                "PLACEHOLDER",
+				LoadSeconds:          2.5,
+				MaxFeedItems:         10,
 				Feed: feedT{
 					Title:       feedFieldInfoT{Parent: "Demo Title", CaptureGroupIndex: []int8{0}},
 					Link:        feedFieldInfoT{Parent: "https://example.com/", CaptureGroupIndex: []int8{1}},
@@ -73,9 +75,20 @@ func main() {
 		other.PrintError("Failed to unmarshal input XML: "+err.Error(), 1)
 	}

-	rawHTML, err := bthtml.GetFromURL(inputInfo.FlareSolverrURL, inputInfo.Feed.Link.Parent, inputInfo.LoadSeconds)
-	if err != nil {
-		other.PrintError("Failed to get HTML content: "+err.Error(), 1)
+	var rawString string
+	if inputInfo.FlareSolverrEndpoint != "" {
+		rawString, err = websrc.GetByScrape(inputInfo.FlareSolverrEndpoint, inputInfo.Feed.Link.Parent, inputInfo.LoadSeconds)
+		if err != nil {
+			other.PrintError("Failed to scrape HTML content: "+err.Error(), 1)
+		}
+	} else if inputInfo.APIMethodEndpointPayload != "" {
+		apiInfoSplit := strings.Split(inputInfo.APIMethodEndpointPayload, "@")
+		rawString, err = websrc.GetByAPIJson(apiInfoSplit[0], apiInfoSplit[1], []byte(apiInfoSplit[2]))
+		if err != nil {
+			other.PrintError("Failed to get API response: "+err.Error(), 1)
+		}
+	} else {
+		other.PrintError("Either FlareSolverrEndpoint or APIMethodPayload must be specified", 1)
 	}

 	atomXML, err := atom.GetFromString(
@@ -85,7 +98,7 @@ func main() {
 			Description: inputInfo.Feed.Description.Parent,
 			Author:      &feeds.Author{Name: inputInfo.Feed.AuthorName.Parent, Email: inputInfo.Feed.AuthorEmail.Parent},
 		},
-		&rawHTML,
+		&rawString,
 		inputInfo.Regex,
 		atom.IndicesT{
 			TitleC:       inputInfo.Feed.Title.CaptureGroupConnector,
@@ -102,7 +115,7 @@ func main() {
 		inputInfo.MaxFeedItems,
 	)
 	if err != nil {
-		other.PrintError("Failed to generate Atom RSS feed from HTML:\n\n"+rawHTML+"\n\n"+err.Error(), 1)
+		other.PrintError("Failed to generate Atom RSS feed from HTML:\n\n"+rawString+"\n\n"+err.Error(), 1)
 	}
 	fmt.Println(*atomXML)
 }
@@ -0,0 +1,34 @@
+package websrc
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"net/http"
+)
+
+type payloadFlareSolverrT struct {
+	Cmd                    string  `json:"cmd"`
+	URL                    string  `json:"url"`
+	MaxTimeoutMilliseconds int     `json:"maxTimeout"`
+	WaitInSeconds          float32 `json:"waitInSeconds"`
+}
+
+func GetByAPIJson(method, endpoint string, payload []byte) (string, error) {
+	client := &http.Client{}
+
+	req, _ := http.NewRequest(method, endpoint, bytes.NewBuffer(payload))
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", errors.New("unable to make " + method + " request to \"" + endpoint + "\": " + err.Error())
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", errors.New("unable to read \"" + endpoint + "\" response body: " + err.Error())
+	}
+	return string(body), nil
+}
@@ -0,0 +1,23 @@
+package websrc
+
+import (
+	"encoding/json"
+	"errors"
+)
+
+func GetByScrape(flareSolverrEndpoint, webPageURL string, loadSeconds float32) (string, error) {
+	payload, err := json.Marshal(payloadFlareSolverrT{
+		Cmd:                    "request.get",
+		URL:                    webPageURL,
+		MaxTimeoutMilliseconds: 60000,
+		WaitInSeconds:          loadSeconds,
+	})
+	if err != nil {
+		return "", errors.New("unable to marshal FlareSolverr payload to JSON: " + err.Error())
+	}
+	body, err := GetByAPIJson("POST", flareSolverrEndpoint, payload)
+	if err != nil {
+		return "", err
+	}
+	return string(body), nil
+}