do not filter out m3u8

Update yt-dlp to 2025.01.12 from 2024.12.23

.github/workflows/push.yml

@@ -1,6 +1,10 @@
 name: Build on push and PRs
 
-on: [push,pull_request]
+on:
+  push:
+    branches:
+      - "master"
+  pull_request:
 
 jobs:
   build:

Dockerfile

@@ -1,9 +1,9 @@
 # bump: golang /GOLANG_VERSION=([\d.]+)/ docker:golang|^1
 # bump: golang link "Release notes" https://golang.org/doc/devel/release.html
-ARG GOLANG_VERSION=1.22.1
+ARG GOLANG_VERSION=1.23.4
 # bump: yt-dlp /YT_DLP=([\d.-]+)/ https://github.com/yt-dlp/yt-dlp.git|/^\d/|sort
 # bump: yt-dlp link "Release notes" https://github.com/yt-dlp/yt-dlp/releases/tag/$LATEST
-ARG YT_DLP=2024.03.10
+ARG YT_DLP=2025.01.12
 
 FROM golang:$GOLANG_VERSION AS base
 ARG YT_DLP

README.md

@@ -4,7 +4,7 @@ Go wrapper for
 [youtube-dl](https://github.com/ytdl-org/youtube-dl) and
 [yt-dlp](https://github.com/yt-dlp/yt-dlp).
 Currently only tested and developed using yt-dlp.
-API documentation can be found at [godoc.org](https://pkg.go.dev/github.com/wader/goutubedl?tab=doc).
+API documentation can be found at [godoc.org](https://pkg.go.dev/gitea.kaz62.ru/dilap54/goutubedl?tab=doc).
 
 See [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp) for how to
 install and what is recommended to install in addition to yt-dl.
@@ -27,7 +27,7 @@ import (
 	"log"
 	"os"
 
-	"github.com/wader/goutubedl"
+	"gitea.kaz62.ru/dilap54/goutubedl"
 )
 
 func main() {
@@ -53,6 +53,16 @@ func main() {
 See [goutubedl cmd tool](cmd/goutubedl/main.go) or [ydls](https://github.com/wader/ydls)
 for usage examples.
 
+### Default options and cache
+
+#### .netrc
+
+goutubedl by default uses `--netrc` to use `~/.netrc` authentication data.
+
+#### Cache directory
+
+yt-dlp stores various extractor session data to speed up things in `${XDG_CACHE_HOME}/yt-dlp` (usually `~/.cache/yt-dlp`). You might want to preverse this directory if your running things in ephemeral conatiners etc.
+
 ### Development
 
 ```sh

cmd/example/main.go

@@ -6,7 +6,7 @@ import (
 	"log"
 	"os"
 
-	"github.com/wader/goutubedl"
+	"gitea.kaz62.ru/dilap54/goutubedl"
 )
 
 func main() {

cmd/goutubedl/main.go

@@ -9,7 +9,7 @@ import (
 	"os"
 	"os/exec"
 
-	"github.com/wader/goutubedl"
+	"gitea.kaz62.ru/dilap54/goutubedl"
 )
 
 var dumpFlag = flag.Bool("J", false, "Dump JSON")

go.mod

@@ -1,4 +1,4 @@
-module github.com/wader/goutubedl
+module gitea.kaz62.ru/dilap54/goutubedl
 
 go 1.12
 

goutubedl.go

@@ -132,8 +132,9 @@ type Info struct {
 	ThumbnailBytes []byte      `json:"-"`
 	Thumbnails     []Thumbnail `json:"thumbnails"`
 
-	Formats   []Format              `json:"formats"`
+	Formats           []Format              `json:"formats"`
-	Subtitles map[string][]Subtitle `json:"subtitles"`
+	Subtitles         map[string][]Subtitle `json:"subtitles"`
+	AutomaticCaptions map[string][]*Caption `json:"automatic_captions"`
 
 	// Playlist entries if _type is playlist
 	Entries []Info `json:"entries"`
@@ -183,6 +184,14 @@ type Subtitle struct {
 	Bytes []byte `json:"-"`
 }
 
+type Caption struct {
+	Ext      string `json:"ext"`
+	Protocol string `json:"protocol"`
+	URL      string `json:"url"`
+	Name     string `json:"name"`
+	Bytes    []byte `json:"-"`
+}
+
 func (f Format) String() string {
 	return fmt.Sprintf("%s:%s:%s abr:%f vbr:%f tbr:%f",
 		f.FormatID,
@@ -217,20 +226,29 @@ var TypeFromString = map[string]Type{
 
 // Options for New()
 type Options struct {
-	Type               Type
+	Type                    Type
-	PlaylistStart      uint   // --playlist-start
+	PlaylistStart           uint   // --playlist-start
-	PlaylistEnd        uint   // --playlist-end
+	PlaylistEnd             uint   // --playlist-end
-	Downloader         string // --downloader
+	Downloader              string // --downloader
-	DownloadThumbnail  bool
+	DownloadThumbnail       bool
-	DownloadSubtitles  bool
+	DownloadSubtitles       bool
-	DownloadSections   string // --download-sections
+	DownloadSubtitlesLang   []string
+	DownloadSubtitlesFormat string
+	DownloadSections        string // --download-sections
+	Impersonate             string // --impersonate
+
 	ProxyUrl           string // --proxy URL  http://host:port or socks5://host:port
+	UseIPV4            bool   // -4 Make all connections via IPv4
+	Cookies            string // --cookies FILE
 	CookiesFromBrowser string // --cookies-from-browser BROWSER[:FOLDER]
 	DebugLog           Printer
 	StderrFn           func(cmd *exec.Cmd) io.Writer // if not nil, function to get Writer for stderr
 	HTTPClient         *http.Client                  // Client for download thumbnail and subtitles (nil use http.DefaultClient)
 	MergeOutputFormat  string                        // --merge-output-format
 	SortingFormat      string                        // --format-sort
+	ExtractorArgs      string                        // --extractor-args
+	InfoArgs           []string
+	DownloadArgs       []string
 
 	// Set to true if you don't want to use the result.Info structure after the goutubedl.New() call,
 	// so the given URL will be downloaded in a single pass in the DownloadResult.Download() call.
@@ -304,27 +322,51 @@ func infoFromURL(
 		ProbePath(),
 		// see comment below about ignoring errors for playlists
 		"--ignore-errors",
+		// TODO: deprecated in yt-dlp?
 		"--no-call-home",
-		"--no-cache-dir",
+		// use safer output filenmaes
-		"--skip-download",
+		// TODO: needed?
 		"--restrict-filenames",
-		// provide URL via stdin for security, youtube-dl has some run command args
+		// use .netrc authentication data
+		"--netrc",
+		// provide url via stdin for security, youtube-dl has some run command args
 		"--batch-file", "-",
-		"-J",
+		// dump info json
+		"--dump-single-json",
 	)
 
 	if options.ProxyUrl != "" {
 		cmd.Args = append(cmd.Args, "--proxy", options.ProxyUrl)
 	}
 
+	if options.UseIPV4 {
+		cmd.Args = append(cmd.Args, "-4")
+	}
+
 	if options.Downloader != "" {
 		cmd.Args = append(cmd.Args, "--downloader", options.Downloader)
 	}
 
+	if options.Impersonate != "" {
+		cmd.Args = append(cmd.Args, "--impersonate", options.Impersonate)
+	}
+
+	if options.Cookies != "" {
+		cmd.Args = append(cmd.Args, "--cookies", options.Cookies)
+	}
+
 	if options.CookiesFromBrowser != "" {
 		cmd.Args = append(cmd.Args, "--cookies-from-browser", options.CookiesFromBrowser)
 	}
 
+	if options.ExtractorArgs != "" {
+		cmd.Args = append(cmd.Args,
+			"--extractor-args", options.ExtractorArgs,
+		)
+	}
+
+	cmd.Args = append(cmd.Args, options.InfoArgs...)
+
 	switch options.Type {
 	case TypePlaylist, TypeChannel:
 		cmd.Args = append(cmd.Args, "--yes-playlist")
@@ -445,6 +487,31 @@ func infoFromURL(
 		}
 	}
 
+	for _, lang := range options.DownloadSubtitlesLang {
+		if _, ok := info.AutomaticCaptions[lang]; !ok {
+			continue
+		}
+
+		for _, caption := range info.AutomaticCaptions[lang] {
+			if options.DownloadSubtitlesFormat != "" && caption.Ext != options.DownloadSubtitlesFormat {
+				continue
+			}
+
+			resp, respErr := get(caption.URL)
+			if respErr == nil {
+				buf, err := io.ReadAll(resp.Body)
+				if err != nil {
+					options.DebugLog.Print("err", "download captions "+caption.URL, err)
+				}
+				resp.Body.Close()
+				caption.Bytes = buf
+			} else {
+				options.DebugLog.Print("err", "download captions "+caption.URL, respErr)
+			}
+		}
+
+	}
+
 	if options.DownloadSubtitles {
 		for _, subtitles := range info.Subtitles {
 			for i, subtitle := range subtitles {
@@ -513,6 +580,8 @@ func (result Result) Download(ctx context.Context, filter string) (*DownloadResu
 }
 
 type DownloadOptions struct {
+	AudioFormats      string // --audio-formats Download audio using formats (best, aac, alac, flac, m4a, mp3, opus, vorbis, wav)
+	DownloadAudioOnly bool   // -x Download audio only from video
 	// Download format matched by filter (usually a format id or quality designator).
 	// If filter is empty, then youtube-dl will use its default format selector.
 	Filter string
@@ -559,12 +628,19 @@ func (result Result) DownloadWithOptions(
 	cmd := exec.CommandContext(
 		ctx,
 		ProbePath(),
-		"--no-call-home",
+		// see comment below about ignoring errors for playlists
-		"--no-cache-dir",
 		"--ignore-errors",
+		// TODO: deprecated in yt-dlp?
+		"--no-call-home",
+		// use non-fancy progress bar
 		"--newline",
+		// use safer output filenmaes
+		// TODO: needed?
 		"--restrict-filenames",
-		"-o", "-",
+		// use .netrc authentication data
+		"--netrc",
+		// write to stdout
+		"--output", "-",
 	)
 
 	if result.Options.noInfoDownload {
@@ -593,7 +669,10 @@ func (result Result) DownloadWithOptions(
 	} else {
 		cmd.Args = append(cmd.Args, "--load-info", jsonTempPath)
 	}
-
+	// force IPV4 Usage
+	if result.Options.UseIPV4 {
+		cmd.Args = append(cmd.Args, "-4")
+	}
 	// don't need to specify if direct as there is only one
 	// also seems to be issues when using filter with generic extractor
 	if !result.Info.Direct && options.Filter != "" {
@@ -604,6 +683,14 @@ func (result Result) DownloadWithOptions(
 		cmd.Args = append(cmd.Args, "--playlist-items", fmt.Sprint(options.PlaylistIndex))
 	}
 
+	if options.DownloadAudioOnly {
+		cmd.Args = append(cmd.Args, "-x")
+	}
+
+	if options.AudioFormats != "" {
+		cmd.Args = append(cmd.Args, "--audio-format", options.AudioFormats)
+	}
+
 	if result.Options.ProxyUrl != "" {
 		cmd.Args = append(cmd.Args, "--proxy", result.Options.ProxyUrl)
 	}
@@ -632,6 +719,14 @@ func (result Result) DownloadWithOptions(
 		)
 	}
 
+	if result.Options.ExtractorArgs != "" {
+		cmd.Args = append(cmd.Args,
+			"--extractor-args", result.Options.ExtractorArgs,
+		)
+	}
+
+	cmd.Args = append(cmd.Args, result.Options.DownloadArgs...)
+
 	cmd.Dir = tempPath
 	var stdoutW io.WriteCloser
 	var stderrW io.WriteCloser

goutubedl_test.go

@@ -16,13 +16,13 @@ import (
 	"strings"
 	"testing"
 
+	"gitea.kaz62.ru/dilap54/goutubedl"
 	"github.com/fortytw2/leaktest"
-	"github.com/wader/goutubedl"
 	"github.com/wader/osleaktest"
 )
 
 const (
-	testVideoRawURL          = "https://www.youtube.com/watch?v=C0DPdy98e4c"
+	testVideoRawURL          = "https://vimeo.com/454525548"
 	playlistRawURL           = "https://soundcloud.com/mattheis/sets/kindred-phenomena"
 	channelRawURL            = "https://www.youtube.com/channel/UCHDm-DKoMyJxKVgwGmuTaQA"
 	subtitlesTestVideoRawURL = "https://www.youtube.com/watch?v=QRS8MkLhQmM"
@@ -139,7 +139,7 @@ func TestParseInfo(t *testing.T) {
 	}{
 		{"https://soundcloud.com/avalonemerson/avalon-emerson-live-at-printworks-london-march-2017", "Avalon Emerson Live at Printworks London 2017"},
 		{"https://www.infoq.com/presentations/Simple-Made-Easy", "Simple Made Easy - InfoQ"},
-		{"https://www.youtube.com/watch?v=uVYWQJ5BB_w", "A Radiolab Producer on the Making of a Podcast"},
+		{"https://vimeo.com/454525548", "Sample Video - 3 minutemp4.mp4"},
 	} {
 		t.Run(c.url, func(t *testing.T) {
 			defer leakChecks(t)()
@@ -221,6 +221,8 @@ func TestPlaylist(t *testing.T) {
 }
 
 func TestChannel(t *testing.T) {
+	t.Skip("skip youtube for now")
+
 	defer leakChecks(t)()
 
 	ydlResult, ydlResultErr := goutubedl.New(
@@ -267,6 +269,8 @@ func TestUnsupportedURL(t *testing.T) {
 }
 
 func TestPlaylistWithPrivateVideo(t *testing.T) {
+	t.Skip("skip youtube for now")
+
 	defer leaktest.Check(t)()
 
 	playlistRawURL := "https://www.youtube.com/playlist?list=PLX0g748fkegS54oiDN4AXKl7BR7mLIydP"
@@ -287,6 +291,8 @@ func TestPlaylistWithPrivateVideo(t *testing.T) {
 }
 
 func TestSubtitles(t *testing.T) {
+	t.Skip("skip youtube for now")
+
 	defer leakChecks(t)()
 
 	ydlResult, ydlResultErr := goutubedl.New(
@@ -332,7 +338,7 @@ func TestDownloadSections(t *testing.T) {
 
 	ydlResult, ydlResultErr := goutubedl.New(
 		context.Background(),
-		"https://www.youtube.com/watch?v=OyuL5biOQ94",
+		"https://vimeo.com/454525548",
 		goutubedl.Options{
 			DownloadSections: fmt.Sprintf("*0:0-0:%d", duration),
 		})
@@ -343,7 +349,7 @@ func TestDownloadSections(t *testing.T) {
 	if ydlResultErr != nil {
 		t.Errorf("failed to download: %s", ydlResultErr)
 	}
-	dr, err := ydlResult.Download(context.Background(), "best")
+	dr, err := ydlResult.Download(context.Background(), "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -554,6 +560,8 @@ func TestDownloadPlaylistEntry(t *testing.T) {
 }
 
 func TestFormatDownloadError(t *testing.T) {
+	t.Skip("test URL broken")
+
 	defer leaktest.Check(t)()
 
 	ydl, ydlErr := goutubedl.New(