From 8672aadcb8b5d987be0e9aa4a6fef26242e7b68d Mon Sep 17 00:00:00 2001 From: Randall Winkhart Date: Sun, 4 Jan 2026 15:36:56 -0500 Subject: [PATCH] Add support for APIs (in addition to scraping) --- bthtml/html.go | 50 ------------------- .../input_xmls/chorus-encore-api_json.xml | 33 ++++++++++++ ...us-encore.xml => chorus-encore-scrape.xml} | 2 +- main.go | 43 ++++++++++------ websrc/apiJSON.go | 34 +++++++++++++ websrc/scrape.go | 23 +++++++++ 6 files changed, 119 insertions(+), 66 deletions(-) delete mode 100644 bthtml/html.go create mode 100644 examples/input_xmls/chorus-encore-api_json.xml rename examples/input_xmls/{chorus-encore.xml => chorus-encore-scrape.xml} (95%) create mode 100644 websrc/apiJSON.go create mode 100644 websrc/scrape.go diff --git a/bthtml/html.go b/bthtml/html.go deleted file mode 100644 index b74465e..0000000 --- a/bthtml/html.go +++ /dev/null @@ -1,50 +0,0 @@ -package bthtml - -import ( - "bytes" - "encoding/json" - "errors" - "io" - "net/http" -) - -type payloadT struct { - Cmd string `json:"cmd"` - URL string `json:"url"` - MaxTimeoutMilliseconds int `json:"maxTimeout"` - WaitInSeconds float32 `json:"waitInSeconds"` -} - -func GetFromURL(flareSolverrURL, webPageURL string, loadSeconds float32) (string, error) { - client := &http.Client{} - - payload := payloadT{ - Cmd: "request.get", - URL: webPageURL, - MaxTimeoutMilliseconds: 60000, - WaitInSeconds: loadSeconds, - } - payloadBytes, err := json.Marshal(payload) - if err != nil { - return "", errors.New("unable to marshal FlareSolverr payload to JSON: " + err.Error()) - } - - req, _ := http.NewRequest( - "POST", - flareSolverrURL, - bytes.NewBuffer(payloadBytes), - ) - req.Header.Set("Content-Type", "application/json") - - resp, err := client.Do(req) - if err != nil { - return "", errors.New("unable to make FlareSolverr request to " + flareSolverrURL + ": " + err.Error()) - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", errors.New("unable to read FlareSolverr response body: " + err.Error()) - } - return string(body), nil -} diff --git a/examples/input_xmls/chorus-encore-api_json.xml b/examples/input_xmls/chorus-encore-api_json.xml new file mode 100644 index 0000000..b4f841e --- /dev/null +++ b/examples/input_xmls/chorus-encore-api_json.xml @@ -0,0 +1,33 @@ + + + + 1 + 10 + + + <parent>Chorus Encore</parent> + <captureGroupConnector /> + <captureGroupIndex>1</captureGroupIndex> + <captureGroupConnector> by </captureGroupConnector> + <captureGroupIndex>2</captureGroupIndex> + + https://www.enchor.us/ + https://www.enchor.us/chart/ + 4 + + + Latest YARG/CH charts published to Chorus Encore + Charter: + 5 + | Album: + 3 + + + Gamers + 5 + + + + diff --git a/examples/input_xmls/chorus-encore.xml b/examples/input_xmls/chorus-encore-scrape.xml similarity index 95% rename from examples/input_xmls/chorus-encore.xml rename to examples/input_xmls/chorus-encore-scrape.xml index f8b1e40..f43e2e8 100644 --- a/examples/input_xmls/chorus-encore.xml +++ b/examples/input_xmls/chorus-encore-scrape.xml @@ -1,5 +1,5 @@ - http://PLACEHOLDER:8191/v1 + http://PLACEHOLDER:8191/v1
\\"Album
(.*?)<\/div>
Album:<\/span> (.*?)<\/div>
Genre:<\/span> .*?Charter:<\/div>(.*?)<\/a><\/div>.*?for=\\"downloadVideos_(.*?)\\"]]> 1 diff --git a/main.go b/main.go index f3392a9..2768acb 100644 --- a/main.go +++ b/main.go @@ -5,10 +5,11 @@ import ( "encoding/xml" "fmt" "os" + "strings" "github.com/gorilla/feeds" "github.com/rwinkhart/backtone/atom" - "github.com/rwinkhart/backtone/bthtml" + "github.com/rwinkhart/backtone/websrc" "github.com/rwinkhart/go-boilerplate/other" ) @@ -32,11 +33,12 @@ type feedT struct { AuthorEmail feedFieldInfoT `xml:"authorEmail"` } type inputT struct { - FlareSolverrURL string `xml:"flareSolverrURL"` - Regex string `xml:"regex"` - LoadSeconds float32 `xml:"loadSeconds"` - MaxFeedItems int `xml:"maxFeedItems"` - Feed feedT `xml:"feed"` + FlareSolverrEndpoint string `xml:"flareSolverrEndpoint,omitempty"` + APIMethodEndpointPayload string `xml:"apiMethodEndpointPayload,omitempty"` + Regex string `xml:"regex"` + LoadSeconds float32 `xml:"loadSeconds"` + MaxFeedItems int `xml:"maxFeedItems"` + Feed feedT `xml:"feed"` } func main() { @@ -44,10 +46,10 @@ func main() { if len(os.Args) < 2 { demoInputBytes, _ := xml.MarshalIndent( inputT{ - FlareSolverrURL: "http://127.0.0.1:8191", - Regex: "PLACEHOLDER", - LoadSeconds: 2.5, - MaxFeedItems: 10, + FlareSolverrEndpoint: "http://127.0.0.1:8191", + Regex: "PLACEHOLDER", + LoadSeconds: 2.5, + MaxFeedItems: 10, Feed: feedT{ Title: feedFieldInfoT{Parent: "Demo Title", CaptureGroupIndex: []int8{0}}, Link: feedFieldInfoT{Parent: "https://example.com/", CaptureGroupIndex: []int8{1}}, @@ -73,9 +75,20 @@ func main() { other.PrintError("Failed to unmarshal input XML: "+err.Error(), 1) } - rawHTML, err := bthtml.GetFromURL(inputInfo.FlareSolverrURL, inputInfo.Feed.Link.Parent, inputInfo.LoadSeconds) - if err != nil { - other.PrintError("Failed to get HTML content: "+err.Error(), 1) + var rawString string + if inputInfo.FlareSolverrEndpoint != "" { + rawString, err = websrc.GetByScrape(inputInfo.FlareSolverrEndpoint, inputInfo.Feed.Link.Parent, inputInfo.LoadSeconds) + if err != nil { + other.PrintError("Failed to scrape HTML content: "+err.Error(), 1) + } + } else if inputInfo.APIMethodEndpointPayload != "" { + apiInfoSplit := strings.Split(inputInfo.APIMethodEndpointPayload, "@") + rawString, err = websrc.GetByAPIJson(apiInfoSplit[0], apiInfoSplit[1], []byte(apiInfoSplit[2])) + if err != nil { + other.PrintError("Failed to get API response: "+err.Error(), 1) + } + } else { + other.PrintError("Either FlareSolverrEndpoint or APIMethodPayload must be specified", 1) } atomXML, err := atom.GetFromString( @@ -85,7 +98,7 @@ func main() { Description: inputInfo.Feed.Description.Parent, Author: &feeds.Author{Name: inputInfo.Feed.AuthorName.Parent, Email: inputInfo.Feed.AuthorEmail.Parent}, }, - &rawHTML, + &rawString, inputInfo.Regex, atom.IndicesT{ TitleC: inputInfo.Feed.Title.CaptureGroupConnector, @@ -102,7 +115,7 @@ func main() { inputInfo.MaxFeedItems, ) if err != nil { - other.PrintError("Failed to generate Atom RSS feed from HTML:\n\n"+rawHTML+"\n\n"+err.Error(), 1) + other.PrintError("Failed to generate Atom RSS feed from HTML:\n\n"+rawString+"\n\n"+err.Error(), 1) } fmt.Println(*atomXML) } diff --git a/websrc/apiJSON.go b/websrc/apiJSON.go new file mode 100644 index 0000000..5d278d6 --- /dev/null +++ b/websrc/apiJSON.go @@ -0,0 +1,34 @@ +package websrc + +import ( + "bytes" + "errors" + "io" + "net/http" +) + +type payloadFlareSolverrT struct { + Cmd string `json:"cmd"` + URL string `json:"url"` + MaxTimeoutMilliseconds int `json:"maxTimeout"` + WaitInSeconds float32 `json:"waitInSeconds"` +} + +func GetByAPIJson(method, endpoint string, payload []byte) (string, error) { + client := &http.Client{} + + req, _ := http.NewRequest(method, endpoint, bytes.NewBuffer(payload)) + req.Header.Set("Content-Type", "application/json") + + resp, err := client.Do(req) + if err != nil { + return "", errors.New("unable to make " + method + " request to \"" + endpoint + "\": " + err.Error()) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", errors.New("unable to read \"" + endpoint + "\" response body: " + err.Error()) + } + return string(body), nil +} diff --git a/websrc/scrape.go b/websrc/scrape.go new file mode 100644 index 0000000..b731837 --- /dev/null +++ b/websrc/scrape.go @@ -0,0 +1,23 @@ +package websrc + +import ( + "encoding/json" + "errors" +) + +func GetByScrape(flareSolverrEndpoint, webPageURL string, loadSeconds float32) (string, error) { + payload, err := json.Marshal(payloadFlareSolverrT{ + Cmd: "request.get", + URL: webPageURL, + MaxTimeoutMilliseconds: 60000, + WaitInSeconds: loadSeconds, + }) + if err != nil { + return "", errors.New("unable to marshal FlareSolverr payload to JSON: " + err.Error()) + } + body, err := GetByAPIJson("POST", flareSolverrEndpoint, payload) + if err != nil { + return "", err + } + return string(body), nil +}