From 7a4ef87470bc787cb99b4ca5658174a7f6463898 Mon Sep 17 00:00:00 2001 From: Randall Winkhart Date: Sun, 4 Jan 2026 05:37:43 -0500 Subject: [PATCH] Initial code commit --- .gitignore | 2 + LICENSE | 2 + README.md | 17 ++++ atom/rss.go | 79 +++++++++++++++++++ bthtml/html.go | 50 ++++++++++++ examples/Caddyfile | 22 ++++++ examples/feed.php | 8 ++ examples/input_xmls/chorus-encore.xml | 1 + go.mod | 7 ++ go.sum | 10 +++ main.go | 108 ++++++++++++++++++++++++++ 11 files changed, 306 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 atom/rss.go create mode 100644 bthtml/html.go create mode 100644 examples/Caddyfile create mode 100644 examples/feed.php create mode 100644 examples/input_xmls/chorus-encore.xml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3fab237 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/backtone +/backtone.exe diff --git a/LICENSE b/LICENSE index f288702..2336df0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,8 @@ GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 + Copyright (c) 2026 Randall Winkhart + Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0f0109f --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# Backtone +Backtone is a program that converts websites to Atom RSS +feeds and prints the results to stdout. It should be called via a +server-side script that then writes the resulting XML to the webpage. + +The name is a playful nod toward converting contents back to the web of yesteryear. +Why suffer in Web 2/3 when you could go "back to one"? + +# Dependencies +To ensure Backtone always supports current ECMAScript standards, it needs a headless browser instance to execute client-side scripts. To avoid Cloudflare/similar challenge interference, we use [FlareSolverr](https://developer.chrome.com/blog/chrome-headless-shell). Its Docker image also contains the browser we use. + +# Usage +All files needed for usage, including an the input XML, server-side script, and reverse proxy config, have examples in the `examples` folder in this repo. + +1. Create a server-side script that calls `backtone` with the only argument being a base64-encoded XML in the format specified when running `backbone` with no arguments +2. Serve that script via a reverse proxy + - Now when a feed reader fetches the feed, the feed will be created in real time diff --git a/atom/rss.go b/atom/rss.go new file mode 100644 index 0000000..0627b5f --- /dev/null +++ b/atom/rss.go @@ -0,0 +1,79 @@ +package atom + +import ( + "errors" + "regexp" + "strconv" + "strings" + "time" + + "github.com/gorilla/feeds" +) + +// Indices indicates which capture group in the +// regex-parsed HTML to reference for each Atom field. +// An empty slice will use a blank/default value. +// For the slices of strings, these will be appended between +// captured group text. +type IndicesT struct { + TitleC []string + TitleI []int8 + LinkC []string + LinkI []int8 + DescriptionC []string + DescriptionI []int8 + AuthorNameC []string + AuthorNameI []int8 + AuthorEmailC []string + AuthorEmailI []int8 +} + +func GetFromHTML(feed *feeds.Feed, rawHTML *string, regexString string, infoIndices IndicesT, maxFeedItems int) (*string, error) { + now := time.Now() + r := regexp.MustCompile(regexString) + newsItems := r.FindAllString(*rawHTML, 5) + for i, item := range newsItems { + info := r.FindStringSubmatch(item) + link := stitchFields(info, infoIndices.LinkC, infoIndices.LinkI) + feedItem := &feeds.Item{ + Title: stitchFields(info, infoIndices.TitleC, infoIndices.TitleI), + Link: &feeds.Link{Href: link}, + Id: "urn:backtoneid:" + strconv.Itoa(i) + ":" + link, + Description: stitchFields(info, infoIndices.DescriptionC, infoIndices.DescriptionI), + Author: &feeds.Author{ + Name: stitchFields(info, infoIndices.AuthorNameC, infoIndices.AuthorNameI), + Email: stitchFields(info, infoIndices.AuthorEmailC, infoIndices.AuthorEmailI), + }, + Created: now, + } + feed.Items = append(feed.Items, feedItem) + if i == maxFeedItems { + break + } + } + atom, err := feed.ToAtom() + if err != nil { + return nil, errors.New("unable to create Atom RSS feed: " + err.Error()) + } + return &atom, nil +} + +func stitchFields(info []string, connectors []string, indices []int8) string { + var output strings.Builder + if len(connectors) > 0 { + for i := range connectors { + output.WriteString(connectors[i]) + if len(indices) >= i+1 { + output.WriteString(info[indices[i]]) + } + } + } else { + for i := range indices { + if i > 0 { + output.WriteString(" | ") + } + output.WriteString(info[indices[i]]) + } + } + return output.String() +} diff --git a/bthtml/html.go b/bthtml/html.go new file mode 100644 index 0000000..b74465e --- /dev/null +++ b/bthtml/html.go @@ -0,0 +1,50 @@ +package bthtml + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" +) + +type payloadT struct { + Cmd string `json:"cmd"` + URL string `json:"url"` + MaxTimeoutMilliseconds int `json:"maxTimeout"` + WaitInSeconds float32 `json:"waitInSeconds"` +} + +func GetFromURL(flareSolverrURL, webPageURL string, loadSeconds float32) (string, error) { + client := &http.Client{} + + payload := payloadT{ + Cmd: "request.get", + URL: webPageURL, + MaxTimeoutMilliseconds: 60000, + WaitInSeconds: loadSeconds, + } + payloadBytes, err := json.Marshal(payload) + if err != nil { + return "", errors.New("unable to marshal FlareSolverr payload to JSON: " + err.Error()) + } + + req, _ := http.NewRequest( + "POST", + flareSolverrURL, + bytes.NewBuffer(payloadBytes), + ) + req.Header.Set("Content-Type", "application/json") + + resp, err := client.Do(req) + if err != nil { + return "", errors.New("unable to make FlareSolverr request to " + flareSolverrURL + ": " + err.Error()) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", errors.New("unable to read FlareSolverr response body: " + err.Error()) + } + return string(body), nil +} diff --git a/examples/Caddyfile b/examples/Caddyfile new file mode 100644 index 0000000..d95a064 --- /dev/null +++ b/examples/Caddyfile @@ -0,0 +1,22 @@ +backtone.example.com { + + tls { + protocols tls1.3 + } + + log { + output file /tmp/caddy-backtone.log { + roll_size 2MiB + roll_keep 3 + roll_keep_for 24h + } + level INFO + } + + root * /path/to/feeds/directory + encode gzip + file_server + + php_fastcgi 127.0.0.1:9000 + +} diff --git a/examples/feed.php b/examples/feed.php new file mode 100644 index 0000000..270fc5a --- /dev/null +++ b/examples/feed.php @@ -0,0 +1,8 @@ + diff --git a/examples/input_xmls/chorus-encore.xml b/examples/input_xmls/chorus-encore.xml new file mode 100644 index 0000000..8b1c789 --- /dev/null +++ b/examples/input_xmls/chorus-encore.xml @@ -0,0 +1 @@ +1
\\"Album
(.*?)<\/div>
Album:<\/span> (.*?)<\/div>
Genre:<\/span> .*?<\/div>
Year:<\/span> .*?<\/div><\/div><\/div>
<\/i><\/i><\/div>
<\/div><\/div>
Charter:<\/div>(.*?)<\/a><\/div>.*?for=\\"downloadVideos_(.*?)\\"]]>http://PLACEHOLDER:8191/v110<parent>Chorus Encore</parent><captureGroupIndex>1</captureGroupIndex>https://www.enchor.us/https://www.enchor.us/chart/4Latest YARG/CH charts published to Chorus EncoreCharter: 3 | Album: 2Gamers3 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8ba1c09 --- /dev/null +++ b/go.mod @@ -0,0 +1,7 @@ +module github.com/rwinkhart/backtone + +go 1.25.5 + +require github.com/rwinkhart/go-boilerplate v0.1.0 + +require github.com/gorilla/feeds v1.2.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b9fbeaa --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/gorilla/feeds v1.2.0 h1:O6pBiXJ5JHhPvqy53NsjKOThq+dNFm8+DFrxBEdzSCc= +github.com/gorilla/feeds v1.2.0/go.mod h1:WMib8uJP3BbY+X8Szd1rA5Pzhdfh+HCCAYT2z7Fza6Y= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rwinkhart/go-boilerplate v0.1.0 h1:EzlVj6R7Bxtl79Nl7R5zRcg6s+Cf2FAqGIzR4giWTQg= +github.com/rwinkhart/go-boilerplate v0.1.0/go.mod h1:cnzIF45I0FCOvE4YIB+26pLCUx2kWyY2llKYZruNaRY= diff --git a/main.go b/main.go new file mode 100644 index 0000000..dc3a342 --- /dev/null +++ b/main.go @@ -0,0 +1,108 @@ +package main + +import ( + "encoding/base64" + "encoding/xml" + "fmt" + "os" + + "github.com/gorilla/feeds" + "github.com/rwinkhart/backtone/atom" + "github.com/rwinkhart/backtone/bthtml" + "github.com/rwinkhart/go-boilerplate/other" +) + +const cfgPath = "/etc/backtone.xml" + +// feedFieldInfo provides the string value for a given field in the parent feed +// and specifies which index in the regex capture group to extract the same field +// from per individual news item. +// An index of -1 can be used to use the same value as the parent feed. +// An index of -2 can be used to use a default (placeholder) value. +type feedFieldInfoT struct { + Parent string `xml:"parent,omitempty"` + CaptureGroupConnector []string `xml:"captureGroupConnector,omitempty"` + CaptureGroupIndex []int8 `xml:"captureGroupIndex,omitempty"` +} +type feedT struct { + Title feedFieldInfoT `xml:"title"` + Link feedFieldInfoT `xml:"link"` + Description feedFieldInfoT `xml:"description"` + AuthorName feedFieldInfoT `xml:"authorName"` + AuthorEmail feedFieldInfoT `xml:"authorEmail"` +} +type inputT struct { + LoadSeconds float32 `xml:"loadSeconds"` + Regex string `xml:"regex"` + FlareSolverrURL string `xml:"flareSolverrURL"` + MaxFeedItems int `xml:"maxFeedItems"` + Feed feedT `xml:"feed"` +} + +func main() { + // handle input argument (XML) + if len(os.Args) < 2 { + demoInputBytes, _ := xml.MarshalIndent( + inputT{ + LoadSeconds: 2.5, + Regex: "PLACEHOLDER", + FlareSolverrURL: "http://127.0.0.1:8191", + MaxFeedItems: 10, + Feed: feedT{ + Title: feedFieldInfoT{Parent: "Demo Title", CaptureGroupIndex: []int8{0}}, + Link: feedFieldInfoT{Parent: "https://example.com/", CaptureGroupIndex: []int8{1}}, + Description: feedFieldInfoT{Parent: "Demo description", CaptureGroupConnector: []string{"Prefix - ", " - Middle - ", " - Suffix"}, CaptureGroupIndex: []int8{1, 3}}, + AuthorName: feedFieldInfoT{Parent: "John Doe", CaptureGroupIndex: []int8{3}}, + AuthorEmail: feedFieldInfoT{Parent: "johndoe@example.com", CaptureGroupIndex: []int8{4}}, + }, + }, + "", + " ", + ) + other.PrintError(os.Args[0]+" must be called with valid base64-encoded XML input using the following schema:\n\n"+string(demoInputBytes), 1) + } + + // decode base64 input XML + decodedXML, err := base64.StdEncoding.DecodeString(os.Args[1]) + if err != nil { + other.PrintError("Failed to decode base64 input XML: "+err.Error(), 1) + } + + var inputInfo inputT + if err := xml.Unmarshal(decodedXML, &inputInfo); err != nil { + other.PrintError("Failed to unmarshal input XML: "+err.Error(), 1) + } + + rawHTML, err := bthtml.GetFromURL(inputInfo.FlareSolverrURL, inputInfo.Feed.Link.Parent, inputInfo.LoadSeconds) + if err != nil { + other.PrintError("Failed to get HTML content: "+err.Error(), 1) + } + + atomXML, err := atom.GetFromHTML( + &feeds.Feed{ + Title: inputInfo.Feed.Title.Parent, + Link: &feeds.Link{Href: inputInfo.Feed.Link.Parent}, + Description: inputInfo.Feed.Description.Parent, + Author: &feeds.Author{Name: inputInfo.Feed.AuthorName.Parent, Email: inputInfo.Feed.AuthorEmail.Parent}, + }, + &rawHTML, + inputInfo.Regex, + atom.IndicesT{ + TitleC: inputInfo.Feed.Title.CaptureGroupConnector, + TitleI: inputInfo.Feed.Title.CaptureGroupIndex, + LinkC: inputInfo.Feed.Link.CaptureGroupConnector, + LinkI: inputInfo.Feed.Link.CaptureGroupIndex, + DescriptionC: inputInfo.Feed.Description.CaptureGroupConnector, + DescriptionI: inputInfo.Feed.Description.CaptureGroupIndex, + AuthorNameC: inputInfo.Feed.AuthorName.CaptureGroupConnector, + AuthorNameI: inputInfo.Feed.AuthorName.CaptureGroupIndex, + AuthorEmailC: inputInfo.Feed.AuthorEmail.CaptureGroupConnector, + AuthorEmailI: inputInfo.Feed.AuthorEmail.CaptureGroupIndex, + }, + inputInfo.MaxFeedItems, + ) + if err != nil { + other.PrintError("Failed to generate Atom RSS feed from HTML:\n\n"+rawHTML+"\n\n"+err.Error(), 1) + } + fmt.Println(*atomXML) +}