diff --git a/bthtml/html.go b/bthtml/html.go
deleted file mode 100644
index b74465e..0000000
--- a/bthtml/html.go
+++ /dev/null
@@ -1,50 +0,0 @@
-package bthtml
-
-import (
- "bytes"
- "encoding/json"
- "errors"
- "io"
- "net/http"
-)
-
-type payloadT struct {
- Cmd string `json:"cmd"`
- URL string `json:"url"`
- MaxTimeoutMilliseconds int `json:"maxTimeout"`
- WaitInSeconds float32 `json:"waitInSeconds"`
-}
-
-func GetFromURL(flareSolverrURL, webPageURL string, loadSeconds float32) (string, error) {
- client := &http.Client{}
-
- payload := payloadT{
- Cmd: "request.get",
- URL: webPageURL,
- MaxTimeoutMilliseconds: 60000,
- WaitInSeconds: loadSeconds,
- }
- payloadBytes, err := json.Marshal(payload)
- if err != nil {
- return "", errors.New("unable to marshal FlareSolverr payload to JSON: " + err.Error())
- }
-
- req, _ := http.NewRequest(
- "POST",
- flareSolverrURL,
- bytes.NewBuffer(payloadBytes),
- )
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := client.Do(req)
- if err != nil {
- return "", errors.New("unable to make FlareSolverr request to " + flareSolverrURL + ": " + err.Error())
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return "", errors.New("unable to read FlareSolverr response body: " + err.Error())
- }
- return string(body), nil
-}
diff --git a/examples/input_xmls/chorus-encore-api_json.xml b/examples/input_xmls/chorus-encore-api_json.xml
new file mode 100644
index 0000000..b4f841e
--- /dev/null
+++ b/examples/input_xmls/chorus-encore-api_json.xml
@@ -0,0 +1,33 @@
+
+
+
+ 1
+ 10
+
+
+ <parent>Chorus Encore</parent>
+ <captureGroupConnector />
+ <captureGroupIndex>1</captureGroupIndex>
+ <captureGroupConnector> by </captureGroupConnector>
+ <captureGroupIndex>2</captureGroupIndex>
+
+ https://www.enchor.us/
+ https://www.enchor.us/chart/
+ 4
+
+
+ Latest YARG/CH charts published to Chorus Encore
+ Charter:
+ 5
+ | Album:
+ 3
+
+
+ Gamers
+ 5
+
+
+
+
diff --git a/examples/input_xmls/chorus-encore.xml b/examples/input_xmls/chorus-encore-scrape.xml
similarity index 95%
rename from examples/input_xmls/chorus-encore.xml
rename to examples/input_xmls/chorus-encore-scrape.xml
index f8b1e40..f43e2e8 100644
--- a/examples/input_xmls/chorus-encore.xml
+++ b/examples/input_xmls/chorus-encore-scrape.xml
@@ -1,5 +1,5 @@
- http://PLACEHOLDER:8191/v1
+ http://PLACEHOLDER:8191/v1
![\\"Album](\\".*?\\")
(.*?)<\/div>
Album:<\/span> (.*?)<\/div>Genre:<\/span> .*?Charter:<\/div>(.*?)<\/a><\/div>.*?for=\\"downloadVideos_(.*?)\\"]]>
1
diff --git a/main.go b/main.go
index f3392a9..2768acb 100644
--- a/main.go
+++ b/main.go
@@ -5,10 +5,11 @@ import (
"encoding/xml"
"fmt"
"os"
+ "strings"
"github.com/gorilla/feeds"
"github.com/rwinkhart/backtone/atom"
- "github.com/rwinkhart/backtone/bthtml"
+ "github.com/rwinkhart/backtone/websrc"
"github.com/rwinkhart/go-boilerplate/other"
)
@@ -32,11 +33,12 @@ type feedT struct {
AuthorEmail feedFieldInfoT `xml:"authorEmail"`
}
type inputT struct {
- FlareSolverrURL string `xml:"flareSolverrURL"`
- Regex string `xml:"regex"`
- LoadSeconds float32 `xml:"loadSeconds"`
- MaxFeedItems int `xml:"maxFeedItems"`
- Feed feedT `xml:"feed"`
+ FlareSolverrEndpoint string `xml:"flareSolverrEndpoint,omitempty"`
+ APIMethodEndpointPayload string `xml:"apiMethodEndpointPayload,omitempty"`
+ Regex string `xml:"regex"`
+ LoadSeconds float32 `xml:"loadSeconds"`
+ MaxFeedItems int `xml:"maxFeedItems"`
+ Feed feedT `xml:"feed"`
}
func main() {
@@ -44,10 +46,10 @@ func main() {
if len(os.Args) < 2 {
demoInputBytes, _ := xml.MarshalIndent(
inputT{
- FlareSolverrURL: "http://127.0.0.1:8191",
- Regex: "PLACEHOLDER",
- LoadSeconds: 2.5,
- MaxFeedItems: 10,
+ FlareSolverrEndpoint: "http://127.0.0.1:8191",
+ Regex: "PLACEHOLDER",
+ LoadSeconds: 2.5,
+ MaxFeedItems: 10,
Feed: feedT{
Title: feedFieldInfoT{Parent: "Demo Title", CaptureGroupIndex: []int8{0}},
Link: feedFieldInfoT{Parent: "https://example.com/", CaptureGroupIndex: []int8{1}},
@@ -73,9 +75,20 @@ func main() {
other.PrintError("Failed to unmarshal input XML: "+err.Error(), 1)
}
- rawHTML, err := bthtml.GetFromURL(inputInfo.FlareSolverrURL, inputInfo.Feed.Link.Parent, inputInfo.LoadSeconds)
- if err != nil {
- other.PrintError("Failed to get HTML content: "+err.Error(), 1)
+ var rawString string
+ if inputInfo.FlareSolverrEndpoint != "" {
+ rawString, err = websrc.GetByScrape(inputInfo.FlareSolverrEndpoint, inputInfo.Feed.Link.Parent, inputInfo.LoadSeconds)
+ if err != nil {
+ other.PrintError("Failed to scrape HTML content: "+err.Error(), 1)
+ }
+ } else if inputInfo.APIMethodEndpointPayload != "" {
+ apiInfoSplit := strings.Split(inputInfo.APIMethodEndpointPayload, "@")
+ rawString, err = websrc.GetByAPIJson(apiInfoSplit[0], apiInfoSplit[1], []byte(apiInfoSplit[2]))
+ if err != nil {
+ other.PrintError("Failed to get API response: "+err.Error(), 1)
+ }
+ } else {
+ other.PrintError("Either FlareSolverrEndpoint or APIMethodPayload must be specified", 1)
}
atomXML, err := atom.GetFromString(
@@ -85,7 +98,7 @@ func main() {
Description: inputInfo.Feed.Description.Parent,
Author: &feeds.Author{Name: inputInfo.Feed.AuthorName.Parent, Email: inputInfo.Feed.AuthorEmail.Parent},
},
- &rawHTML,
+ &rawString,
inputInfo.Regex,
atom.IndicesT{
TitleC: inputInfo.Feed.Title.CaptureGroupConnector,
@@ -102,7 +115,7 @@ func main() {
inputInfo.MaxFeedItems,
)
if err != nil {
- other.PrintError("Failed to generate Atom RSS feed from HTML:\n\n"+rawHTML+"\n\n"+err.Error(), 1)
+ other.PrintError("Failed to generate Atom RSS feed from HTML:\n\n"+rawString+"\n\n"+err.Error(), 1)
}
fmt.Println(*atomXML)
}
diff --git a/websrc/apiJSON.go b/websrc/apiJSON.go
new file mode 100644
index 0000000..5d278d6
--- /dev/null
+++ b/websrc/apiJSON.go
@@ -0,0 +1,34 @@
+package websrc
+
+import (
+ "bytes"
+ "errors"
+ "io"
+ "net/http"
+)
+
+type payloadFlareSolverrT struct {
+ Cmd string `json:"cmd"`
+ URL string `json:"url"`
+ MaxTimeoutMilliseconds int `json:"maxTimeout"`
+ WaitInSeconds float32 `json:"waitInSeconds"`
+}
+
+func GetByAPIJson(method, endpoint string, payload []byte) (string, error) {
+ client := &http.Client{}
+
+ req, _ := http.NewRequest(method, endpoint, bytes.NewBuffer(payload))
+ req.Header.Set("Content-Type", "application/json")
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return "", errors.New("unable to make " + method + " request to \"" + endpoint + "\": " + err.Error())
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return "", errors.New("unable to read \"" + endpoint + "\" response body: " + err.Error())
+ }
+ return string(body), nil
+}
diff --git a/websrc/scrape.go b/websrc/scrape.go
new file mode 100644
index 0000000..b731837
--- /dev/null
+++ b/websrc/scrape.go
@@ -0,0 +1,23 @@
+package websrc
+
+import (
+ "encoding/json"
+ "errors"
+)
+
+func GetByScrape(flareSolverrEndpoint, webPageURL string, loadSeconds float32) (string, error) {
+ payload, err := json.Marshal(payloadFlareSolverrT{
+ Cmd: "request.get",
+ URL: webPageURL,
+ MaxTimeoutMilliseconds: 60000,
+ WaitInSeconds: loadSeconds,
+ })
+ if err != nil {
+ return "", errors.New("unable to marshal FlareSolverr payload to JSON: " + err.Error())
+ }
+ body, err := GetByAPIJson("POST", flareSolverrEndpoint, payload)
+ if err != nil {
+ return "", err
+ }
+ return string(body), nil
+}