Compare commits
124 Commits
v2.0.0
...
bump-golan
Author | SHA1 | Date | |
---|---|---|---|
40fcd69601 | |||
5d0a261ec7 | |||
e76467b99a | |||
ccf6ee70f7 | |||
1397992b2e | |||
d5bf934f84 | |||
f62b41e2a6 | |||
4a4a53c745 | |||
242d6af088 | |||
5d7e2d8fc4 | |||
131c95e1dd | |||
0986214517 | |||
a52c8dca32 | |||
f4ee4aff9a | |||
b5b253140e | |||
4b7a642b1f | |||
15628e7770 | |||
4b359ddd61 | |||
7993835dad | |||
0846732ece | |||
3eb676f2fd | |||
dfbb1770e4 | |||
3004939371 | |||
79030f02cf | |||
b861e6f3a2 | |||
cb86a6c681 | |||
0ca6990373 | |||
8cef76d0c0 | |||
cad168222b | |||
8598c9e6bb | |||
5cb220b4d7 | |||
74fbd541cd | |||
4dfa55bcfd | |||
fdbd42f666 | |||
2ac97f392a | |||
aba4746463 | |||
1fdc3448b1 | |||
598fce21e7 | |||
ff8aaa0333 | |||
7b1a15b73a | |||
24fd49fe06 | |||
04eb47cbe8 | |||
ae007cbb01 | |||
95260c8d4e | |||
8b5d2141e0 | |||
95b1142af6 | |||
8dad7bee96 | |||
396be61f8d | |||
98ea7a02a4 | |||
c309f999af | |||
47d887d633 | |||
05e5ea7b72 | |||
49794b3db4 | |||
bbd58324d8 | |||
912248327a | |||
3f0fe87c2e | |||
7f2357b6ae | |||
bab64bab72 | |||
0d8b592770 | |||
8b34407df2 | |||
683e9549e7 | |||
01bac34c4c | |||
efe7640980 | |||
adddb11ead | |||
a86f15df0f | |||
7e5db67282 | |||
e3c7c3e3f5 | |||
5019379921 | |||
8ba6398d97 | |||
f88434dfb2 | |||
dbbbde44cb | |||
e00731da32 | |||
8249795c0a | |||
9bed0812c3 | |||
392bd0f15e | |||
307a49a1d3 | |||
c702698079 | |||
782e0b7262 | |||
ab26347454 | |||
02a0a71721 | |||
ed998eb769 | |||
14a15258d0 | |||
5300e997b8 | |||
3f0889f7e8 | |||
12053d10b1 | |||
fc0dec1f1f | |||
427b7fa536 | |||
e431cded03 | |||
fcb3278031 | |||
cfdeb18339 | |||
d87bdbe81c | |||
2438eebf7a | |||
9a0dc509a3 | |||
36319b5e0f | |||
89ddc38573 | |||
e29655f2bd | |||
3184158e02 | |||
89e825670c | |||
b6c1793051 | |||
a230313e47 | |||
ddaf5ad2fa | |||
a46b2837fc | |||
f97b09d082 | |||
fcfd4acd74 | |||
de2e0d1840 | |||
ad9066c4f4 | |||
02ec4fe77d | |||
4b1de1d3ad | |||
f483ae69c8 | |||
c1dcced791 | |||
3c1f4ce6da | |||
2d7d4563d9 | |||
59d0f9fc9e | |||
b2b2500ae9 | |||
b5d5a177c0 | |||
f84c8607fa | |||
ce02413ad4 | |||
20583d83fd | |||
42e6060738 | |||
3d23796f5f | |||
87e9b10e45 | |||
a1ae14605c | |||
64f5dfc85b | |||
4046d6dc4b |
6
.github/workflows/push.yml
vendored
6
.github/workflows/push.yml
vendored
@ -1,6 +1,10 @@
|
||||
name: Build on push and PRs
|
||||
|
||||
on: [push,pull_request]
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
@ -1,9 +1,9 @@
|
||||
# bump: golang /GOLANG_VERSION=([\d.]+)/ docker:golang|^1
|
||||
# bump: golang link "Release notes" https://golang.org/doc/devel/release.html
|
||||
ARG GOLANG_VERSION=1.20.5
|
||||
ARG GOLANG_VERSION=1.23.0
|
||||
# bump: yt-dlp /YT_DLP=([\d.-]+)/ https://github.com/yt-dlp/yt-dlp.git|/^\d/|sort
|
||||
# bump: yt-dlp link "Release notes" https://github.com/yt-dlp/yt-dlp/releases/tag/$LATEST
|
||||
ARG YT_DLP=2023.03.04
|
||||
ARG YT_DLP=2024.08.06
|
||||
|
||||
FROM golang:$GOLANG_VERSION AS base
|
||||
ARG YT_DLP
|
||||
@ -12,7 +12,8 @@ RUN \
|
||||
apt-get update -q && \
|
||||
apt-get install -y -q python-is-python3 && \
|
||||
curl -L https://github.com/yt-dlp/yt-dlp/releases/download/$YT_DLP/yt-dlp -o /usr/local/bin/yt-dlp && \
|
||||
chmod a+x /usr/local/bin/yt-dlp
|
||||
chmod a+x /usr/local/bin/yt-dlp && \
|
||||
apt-get install -y ffmpeg
|
||||
|
||||
FROM base AS dev
|
||||
|
||||
|
29
README.md
29
README.md
@ -1,18 +1,19 @@
|
||||
## goutubedl
|
||||
|
||||
Go wrapper for [youtube-dl](https://github.com/ytdl-org/youtube-dl) and [yt-dlp](https://github.com/yt-dlp/yt-dlp), currently tested and
|
||||
developed using yt-dlp.
|
||||
Go wrapper for
|
||||
[youtube-dl](https://github.com/ytdl-org/youtube-dl) and
|
||||
[yt-dlp](https://github.com/yt-dlp/yt-dlp).
|
||||
Currently only tested and developed using yt-dlp.
|
||||
API documentation can be found at [godoc.org](https://pkg.go.dev/github.com/wader/goutubedl?tab=doc).
|
||||
|
||||
See [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp) for how to
|
||||
install and what is recommended to install in addition to youtube-dl.
|
||||
install and what is recommended to install in addition to yt-dl.
|
||||
|
||||
goutubedl default uses `PATH` to find youtube-dl but it can be configured with the `goutubedl.Path`
|
||||
variable. Default is currently `youtube-dl` for backwards compability. If your using yt-dlp you
|
||||
probably want to set it to `yt-dlp`.
|
||||
goutubedl default uses `PATH` to find `youtube-dl` or `yt-dlp` (in that order) but can be configured with the
|
||||
`goutubedl.Path` variable.
|
||||
|
||||
Due to the nature and frequent updates of youtube-dl only the latest version
|
||||
is tested. But it seems to work well with older versions also.
|
||||
Due to the nature of and frequent updates of yt-dl only the latest version is tested.
|
||||
But it seems to work well with older versions also.
|
||||
|
||||
### Usage
|
||||
|
||||
@ -52,10 +53,20 @@ func main() {
|
||||
See [goutubedl cmd tool](cmd/goutubedl/main.go) or [ydls](https://github.com/wader/ydls)
|
||||
for usage examples.
|
||||
|
||||
### Default options and cache
|
||||
|
||||
#### .netrc
|
||||
|
||||
goutubedl by default uses `--netrc` to use `~/.netrc` authentication data.
|
||||
|
||||
#### Cache directory
|
||||
|
||||
yt-dlp stores various extractor session data to speed up things in `${XDG_CACHE_HOME}/yt-dlp` (usually `~/.cache/yt-dlp`). You might want to preverse this directory if your running things in ephemeral conatiners etc.
|
||||
|
||||
### Development
|
||||
|
||||
```sh
|
||||
docker build -t goutubedl-dev .
|
||||
docker build --target dev -t goutubedl-dev .
|
||||
docker run --rm -ti -v "$PWD:$PWD" -w "$PWD" goutubedl-dev
|
||||
go test -v -race -cover
|
||||
```
|
||||
|
@ -7,6 +7,7 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
||||
"github.com/wader/goutubedl"
|
||||
)
|
||||
@ -24,14 +25,18 @@ func main() {
|
||||
result, err := goutubedl.New(
|
||||
context.Background(),
|
||||
flag.Arg(0),
|
||||
goutubedl.Options{Type: optType, DebugLog: log.Default()},
|
||||
goutubedl.Options{
|
||||
Type: optType,
|
||||
DebugLog: log.Default(),
|
||||
StderrFn: func(cmd *exec.Cmd) io.Writer { return os.Stderr },
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
if *dumpFlag {
|
||||
json.NewEncoder(os.Stdout).Encode(result.Info)
|
||||
_ = json.NewEncoder(os.Stdout).Encode(result.Info)
|
||||
return
|
||||
}
|
||||
|
||||
|
330
goutubedl.go
330
goutubedl.go
@ -9,7 +9,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
@ -18,8 +17,22 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Path to youtube-dl binary. Default look for "youtube-dl" in PATH.
|
||||
var Path = "youtube-dl"
|
||||
// Path to youtube-dl binary. If not set look for "youtube-dl" then "yt-dlp" in PATH.
|
||||
var Path = ""
|
||||
|
||||
func ProbePath() string {
|
||||
if Path != "" {
|
||||
return Path
|
||||
}
|
||||
|
||||
for _, n := range []string{"youtube-dl", "yt-dlp"} {
|
||||
if p, err := exec.LookPath(n); err == nil {
|
||||
return p
|
||||
}
|
||||
}
|
||||
|
||||
return "youtube-dl"
|
||||
}
|
||||
|
||||
// Printer is something that can print
|
||||
type Printer interface {
|
||||
@ -115,7 +128,7 @@ type Info struct {
|
||||
WebpageURL string `json:"webpage_url"`
|
||||
Description string `json:"description"`
|
||||
Thumbnail string `json:"thumbnail"`
|
||||
// not unmarshalled, populated from image thumbnail file
|
||||
// don't unmarshal, populated from image thumbnail file
|
||||
ThumbnailBytes []byte `json:"-"`
|
||||
Thumbnails []Thumbnail `json:"thumbnails"`
|
||||
|
||||
@ -166,7 +179,7 @@ type Subtitle struct {
|
||||
URL string `json:"url"`
|
||||
Ext string `json:"ext"`
|
||||
Language string `json:"-"`
|
||||
// not unmarshalled, populated from subtitle file
|
||||
// don't unmarshal, populated from subtitle file
|
||||
Bytes []byte `json:"-"`
|
||||
}
|
||||
|
||||
@ -191,31 +204,44 @@ const (
|
||||
TypeSingle
|
||||
// TypePlaylist playlist with multiple tracks, files etc
|
||||
TypePlaylist
|
||||
// TypeChannel channel containing one or more playlists, which will be flattened
|
||||
TypeChannel
|
||||
)
|
||||
|
||||
var TypeFromString = map[string]Type{
|
||||
"any": TypeAny,
|
||||
"single": TypeSingle,
|
||||
"playlist": TypePlaylist,
|
||||
"channel": TypeChannel,
|
||||
}
|
||||
|
||||
// Options for New()
|
||||
type Options struct {
|
||||
Type Type
|
||||
PlaylistStart uint // --playlist-start
|
||||
PlaylistEnd uint // --playlist-end
|
||||
DownloadThumbnail bool
|
||||
DownloadSubtitles bool
|
||||
ProxyUrl string // --proxy URL http://host:port or socks5://host:port
|
||||
DebugLog Printer
|
||||
StderrFn func(cmd *exec.Cmd) io.Writer // if not nil, function to get Writer for stderr
|
||||
HTTPClient *http.Client // Client for download thumbnail and subtitles (nil use http.DefaultClient)
|
||||
Type Type
|
||||
PlaylistStart uint // --playlist-start
|
||||
PlaylistEnd uint // --playlist-end
|
||||
Downloader string // --downloader
|
||||
DownloadThumbnail bool
|
||||
DownloadSubtitles bool
|
||||
DownloadSections string // --download-sections
|
||||
ProxyUrl string // --proxy URL http://host:port or socks5://host:port
|
||||
UseIPV4 bool // -4 Make all connections via IPv4
|
||||
CookiesFromBrowser string // --cookies-from-browser BROWSER[:FOLDER]
|
||||
DebugLog Printer
|
||||
StderrFn func(cmd *exec.Cmd) io.Writer // if not nil, function to get Writer for stderr
|
||||
HTTPClient *http.Client // Client for download thumbnail and subtitles (nil use http.DefaultClient)
|
||||
MergeOutputFormat string // --merge-output-format
|
||||
SortingFormat string // --format-sort
|
||||
|
||||
// Set to true if you don't want to use the result.Info structure after the goutubedl.New() call,
|
||||
// so the given URL will be downloaded in a single pass in the DownloadResult.Download() call.
|
||||
noInfoDownload bool
|
||||
}
|
||||
|
||||
// Version of youtube-dl.
|
||||
// Might be a good idea to call at start to assert that youtube-dl can be found.
|
||||
func Version(ctx context.Context) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, Path, "--version")
|
||||
cmd := exec.CommandContext(ctx, ProbePath(), "--version")
|
||||
versionBytes, cmdErr := cmd.Output()
|
||||
if cmdErr != nil {
|
||||
return "", cmdErr
|
||||
@ -224,12 +250,35 @@ func Version(ctx context.Context) (string, error) {
|
||||
return strings.TrimSpace(string(versionBytes)), nil
|
||||
}
|
||||
|
||||
// Downloads given URL using the given options and filter (usually a format id or quality designator).
|
||||
// If filter is empty, then youtube-dl will use its default format selector.
|
||||
func Download(
|
||||
ctx context.Context,
|
||||
rawURL string,
|
||||
options Options,
|
||||
filter string,
|
||||
) (*DownloadResult, error) {
|
||||
options.noInfoDownload = true
|
||||
d, err := New(ctx, rawURL, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d.Download(ctx, filter)
|
||||
}
|
||||
|
||||
// New downloads metadata for URL
|
||||
func New(ctx context.Context, rawURL string, options Options) (result Result, err error) {
|
||||
if options.DebugLog == nil {
|
||||
options.DebugLog = nopPrinter{}
|
||||
}
|
||||
|
||||
if options.noInfoDownload {
|
||||
return Result{
|
||||
RawURL: rawURL,
|
||||
Options: options,
|
||||
}, nil
|
||||
}
|
||||
|
||||
info, rawJSON, err := infoFromURL(ctx, rawURL, options)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
@ -240,31 +289,53 @@ func New(ctx context.Context, rawURL string, options Options) (result Result, er
|
||||
|
||||
return Result{
|
||||
Info: info,
|
||||
RawURL: rawURL,
|
||||
RawJSON: rawJSONCopy,
|
||||
Options: options,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func infoFromURL(ctx context.Context, rawURL string, options Options) (info Info, rawJSON []byte, err error) {
|
||||
func infoFromURL(
|
||||
ctx context.Context,
|
||||
rawURL string,
|
||||
options Options,
|
||||
) (info Info, rawJSON []byte, err error) {
|
||||
cmd := exec.CommandContext(
|
||||
ctx,
|
||||
Path,
|
||||
ProbePath(),
|
||||
// see comment below about ignoring errors for playlists
|
||||
"--ignore-errors",
|
||||
// TODO: deprecated in yt-dlp?
|
||||
"--no-call-home",
|
||||
"--no-cache-dir",
|
||||
"--skip-download",
|
||||
// use safer output filenmaes
|
||||
// TODO: needed?
|
||||
"--restrict-filenames",
|
||||
// provide URL via stdin for security, youtube-dl has some run command args
|
||||
// use .netrc authentication data
|
||||
"--netrc",
|
||||
// provide url via stdin for security, youtube-dl has some run command args
|
||||
"--batch-file", "-",
|
||||
"-J",
|
||||
// dump info json
|
||||
"--dump-single-json",
|
||||
)
|
||||
|
||||
if options.ProxyUrl != "" {
|
||||
cmd.Args = append(cmd.Args, "--proxy", options.ProxyUrl)
|
||||
}
|
||||
|
||||
if options.Type == TypePlaylist {
|
||||
if options.UseIPV4 {
|
||||
cmd.Args = append(cmd.Args, "-4")
|
||||
}
|
||||
|
||||
if options.Downloader != "" {
|
||||
cmd.Args = append(cmd.Args, "--downloader", options.Downloader)
|
||||
}
|
||||
|
||||
if options.CookiesFromBrowser != "" {
|
||||
cmd.Args = append(cmd.Args, "--cookies-from-browser", options.CookiesFromBrowser)
|
||||
}
|
||||
|
||||
switch options.Type {
|
||||
case TypePlaylist, TypeChannel:
|
||||
cmd.Args = append(cmd.Args, "--yes-playlist")
|
||||
|
||||
if options.PlaylistStart > 0 {
|
||||
@ -277,7 +348,7 @@ func infoFromURL(ctx context.Context, rawURL string, options Options) (info Info
|
||||
"--playlist-end", strconv.Itoa(int(options.PlaylistEnd)),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
case TypeSingle:
|
||||
if options.DownloadSubtitles {
|
||||
cmd.Args = append(cmd.Args,
|
||||
"--all-subs",
|
||||
@ -286,14 +357,18 @@ func infoFromURL(ctx context.Context, rawURL string, options Options) (info Info
|
||||
cmd.Args = append(cmd.Args,
|
||||
"--no-playlist",
|
||||
)
|
||||
case TypeAny:
|
||||
break
|
||||
default:
|
||||
return Info{}, nil, fmt.Errorf("unhandled options type value: %d", options.Type)
|
||||
}
|
||||
|
||||
tempPath, _ := ioutil.TempDir("", "ydls")
|
||||
tempPath, _ := os.MkdirTemp("", "ydls")
|
||||
defer os.RemoveAll(tempPath)
|
||||
|
||||
stdoutBuf := &bytes.Buffer{}
|
||||
stderrBuf := &bytes.Buffer{}
|
||||
stderrWriter := ioutil.Discard
|
||||
stderrWriter := io.Discard
|
||||
if options.StderrFn != nil {
|
||||
stderrWriter = options.StderrFn(cmd)
|
||||
}
|
||||
@ -367,7 +442,7 @@ func infoFromURL(ctx context.Context, rawURL string, options Options) (info Info
|
||||
if options.DownloadThumbnail && info.Thumbnail != "" {
|
||||
resp, respErr := get(info.Thumbnail)
|
||||
if respErr == nil {
|
||||
buf, _ := ioutil.ReadAll(resp.Body)
|
||||
buf, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
info.ThumbnailBytes = buf
|
||||
}
|
||||
@ -384,7 +459,7 @@ func infoFromURL(ctx context.Context, rawURL string, options Options) (info Info
|
||||
for i, subtitle := range subtitles {
|
||||
resp, respErr := get(subtitle.URL)
|
||||
if respErr == nil {
|
||||
buf, _ := ioutil.ReadAll(resp.Body)
|
||||
buf, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
subtitles[i].Bytes = buf
|
||||
}
|
||||
@ -393,15 +468,31 @@ func infoFromURL(ctx context.Context, rawURL string, options Options) (info Info
|
||||
}
|
||||
|
||||
// as we ignore errors for playlists some entries might show up as null
|
||||
if options.Type == TypePlaylist {
|
||||
var filteredEntrise []Info
|
||||
//
|
||||
// note: instead of doing full recursion, we assume entries in
|
||||
// playlists and channels are at most 2 levels deep, and we just
|
||||
// collect entries from both levels.
|
||||
//
|
||||
// the following cases have not been tested:
|
||||
//
|
||||
// - entries that are more than 2 levels deep (will be missed)
|
||||
// - the ability to restrict entries to a single level (we include both levels)
|
||||
if options.Type == TypePlaylist || options.Type == TypeChannel {
|
||||
var filteredEntries []Info
|
||||
for _, e := range info.Entries {
|
||||
if e.ID == "" {
|
||||
if e.Type == "playlist" {
|
||||
for _, ee := range e.Entries {
|
||||
if ee.ID == "" {
|
||||
continue
|
||||
}
|
||||
filteredEntries = append(filteredEntries, ee)
|
||||
}
|
||||
continue
|
||||
} else if e.ID != "" {
|
||||
filteredEntries = append(filteredEntries, e)
|
||||
}
|
||||
filteredEntrise = append(filteredEntrise, e)
|
||||
}
|
||||
info.Entries = filteredEntrise
|
||||
info.Entries = filteredEntries
|
||||
}
|
||||
|
||||
return info, stdoutBuf.Bytes(), nil
|
||||
@ -410,6 +501,7 @@ func infoFromURL(ctx context.Context, rawURL string, options Options) (info Info
|
||||
// Result metadata for a URL
|
||||
type Result struct {
|
||||
Info Info
|
||||
RawURL string
|
||||
RawJSON []byte // saved raw JSON. Used later when downloading
|
||||
Options Options // options passed to New
|
||||
}
|
||||
@ -420,24 +512,53 @@ type DownloadResult struct {
|
||||
waitCh chan struct{}
|
||||
}
|
||||
|
||||
// Download format matched by filter (usually a format id or "best").
|
||||
// Filter should not be a combine filter like "1+2" as then youtube-dl
|
||||
// won't write to stdout.
|
||||
// Download format matched by filter (usually a format id or quality designator).
|
||||
// If filter is empty, then youtube-dl will use its default format selector.
|
||||
// It's a shortcut of DownloadWithOptions where the options use the default value
|
||||
func (result Result) Download(ctx context.Context, filter string) (*DownloadResult, error) {
|
||||
return result.DownloadWithOptions(ctx, DownloadOptions{
|
||||
Filter: filter,
|
||||
})
|
||||
}
|
||||
|
||||
type DownloadOptions struct {
|
||||
// Download format matched by filter (usually a format id or quality designator).
|
||||
// If filter is empty, then youtube-dl will use its default format selector.
|
||||
Filter string
|
||||
// The index of the entry to download from the playlist that would be
|
||||
// passed to youtube-dl via --playlist-items. The index value starts at 1
|
||||
PlaylistIndex int
|
||||
}
|
||||
|
||||
func (result Result) DownloadWithOptions(
|
||||
ctx context.Context,
|
||||
options DownloadOptions,
|
||||
) (*DownloadResult, error) {
|
||||
debugLog := result.Options.DebugLog
|
||||
|
||||
if result.Info.Type == "playlist" || result.Info.Type == "multi_video" {
|
||||
return nil, fmt.Errorf("can't download a playlist")
|
||||
if !result.Options.noInfoDownload {
|
||||
if (result.Info.Type == "playlist" ||
|
||||
result.Info.Type == "multi_video" ||
|
||||
result.Info.Type == "channel") &&
|
||||
options.PlaylistIndex == 0 {
|
||||
return nil, fmt.Errorf(
|
||||
"can't download a playlist when the playlist index options is not set",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
tempPath, tempErr := ioutil.TempDir("", "ydls")
|
||||
tempPath, tempErr := os.MkdirTemp("", "ydls")
|
||||
if tempErr != nil {
|
||||
return nil, tempErr
|
||||
}
|
||||
jsonTempPath := path.Join(tempPath, "info.json")
|
||||
if err := ioutil.WriteFile(jsonTempPath, result.RawJSON, 0600); err != nil {
|
||||
os.RemoveAll(tempPath)
|
||||
return nil, err
|
||||
|
||||
var jsonTempPath string
|
||||
if !result.Options.noInfoDownload {
|
||||
jsonTempPath = path.Join(tempPath, "info.json")
|
||||
if err := os.WriteFile(jsonTempPath, result.RawJSON, 0600); err != nil {
|
||||
os.RemoveAll(tempPath)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
dr := &DownloadResult{
|
||||
@ -446,35 +567,102 @@ func (result Result) Download(ctx context.Context, filter string) (*DownloadResu
|
||||
|
||||
cmd := exec.CommandContext(
|
||||
ctx,
|
||||
Path,
|
||||
"--no-call-home",
|
||||
"--no-cache-dir",
|
||||
ProbePath(),
|
||||
// see comment below about ignoring errors for playlists
|
||||
"--ignore-errors",
|
||||
// TODO: deprecated in yt-dlp?
|
||||
"--no-call-home",
|
||||
// use non-fancy progress bar
|
||||
"--newline",
|
||||
// use safer output filenmaes
|
||||
// TODO: needed?
|
||||
"--restrict-filenames",
|
||||
"--load-info", jsonTempPath,
|
||||
"-o", "-",
|
||||
// use .netrc authentication data
|
||||
"--netrc",
|
||||
// write to stdout
|
||||
"--output", "-",
|
||||
)
|
||||
|
||||
if result.Options.noInfoDownload {
|
||||
// provide URL via stdin for security, youtube-dl has some run command args
|
||||
cmd.Args = append(cmd.Args, "--batch-file", "-")
|
||||
cmd.Stdin = bytes.NewBufferString(result.RawURL + "\n")
|
||||
|
||||
if result.Options.Type == TypePlaylist {
|
||||
cmd.Args = append(cmd.Args, "--yes-playlist")
|
||||
|
||||
if result.Options.PlaylistStart > 0 {
|
||||
cmd.Args = append(cmd.Args,
|
||||
"--playlist-start", strconv.Itoa(int(result.Options.PlaylistStart)),
|
||||
)
|
||||
}
|
||||
if result.Options.PlaylistEnd > 0 {
|
||||
cmd.Args = append(cmd.Args,
|
||||
"--playlist-end", strconv.Itoa(int(result.Options.PlaylistEnd)),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
cmd.Args = append(cmd.Args,
|
||||
"--no-playlist",
|
||||
)
|
||||
}
|
||||
} else {
|
||||
cmd.Args = append(cmd.Args, "--load-info", jsonTempPath)
|
||||
}
|
||||
// force IPV4 Usage
|
||||
if result.Options.UseIPV4 {
|
||||
cmd.Args = append(cmd.Args, "-4")
|
||||
}
|
||||
// don't need to specify if direct as there is only one
|
||||
// also seems to be issues when using filter with generic extractor
|
||||
if !result.Info.Direct {
|
||||
cmd.Args = append(cmd.Args, "-f", filter)
|
||||
if !result.Info.Direct && options.Filter != "" {
|
||||
cmd.Args = append(cmd.Args, "-f", options.Filter)
|
||||
}
|
||||
|
||||
if options.PlaylistIndex > 0 {
|
||||
cmd.Args = append(cmd.Args, "--playlist-items", fmt.Sprint(options.PlaylistIndex))
|
||||
}
|
||||
|
||||
if result.Options.ProxyUrl != "" {
|
||||
cmd.Args = append(cmd.Args, "--proxy", result.Options.ProxyUrl)
|
||||
}
|
||||
|
||||
cmd.Dir = tempPath
|
||||
var w io.WriteCloser
|
||||
dr.reader, w = io.Pipe()
|
||||
|
||||
stderrWriter := ioutil.Discard
|
||||
if result.Options.StderrFn != nil {
|
||||
stderrWriter = result.Options.StderrFn(cmd)
|
||||
if result.Options.Downloader != "" {
|
||||
cmd.Args = append(cmd.Args, "--downloader", result.Options.Downloader)
|
||||
}
|
||||
cmd.Stdout = w
|
||||
cmd.Stderr = stderrWriter
|
||||
|
||||
if result.Options.DownloadSections != "" {
|
||||
cmd.Args = append(cmd.Args, "--download-sections", result.Options.DownloadSections)
|
||||
}
|
||||
|
||||
if result.Options.CookiesFromBrowser != "" {
|
||||
cmd.Args = append(cmd.Args, "--cookies-from-browser", result.Options.CookiesFromBrowser)
|
||||
}
|
||||
|
||||
if result.Options.MergeOutputFormat != "" {
|
||||
cmd.Args = append(cmd.Args,
|
||||
"--merge-output-format", result.Options.MergeOutputFormat,
|
||||
)
|
||||
}
|
||||
|
||||
if result.Options.SortingFormat != "" {
|
||||
cmd.Args = append(cmd.Args,
|
||||
"--format-sort", result.Options.SortingFormat,
|
||||
)
|
||||
}
|
||||
|
||||
cmd.Dir = tempPath
|
||||
var stdoutW io.WriteCloser
|
||||
var stderrW io.WriteCloser
|
||||
var stderrR io.Reader
|
||||
dr.reader, stdoutW = io.Pipe()
|
||||
stderrR, stderrW = io.Pipe()
|
||||
optStderrWriter := io.Discard
|
||||
if result.Options.StderrFn != nil {
|
||||
optStderrWriter = result.Options.StderrFn(cmd)
|
||||
}
|
||||
cmd.Stdout = stdoutW
|
||||
cmd.Stderr = io.MultiWriter(optStderrWriter, stderrW)
|
||||
|
||||
debugLog.Print("cmd", " ", cmd.Args)
|
||||
if err := cmd.Start(); err != nil {
|
||||
@ -483,13 +671,33 @@ func (result Result) Download(ctx context.Context, filter string) (*DownloadResu
|
||||
}
|
||||
|
||||
go func() {
|
||||
cmd.Wait()
|
||||
w.Close()
|
||||
_ = cmd.Wait()
|
||||
stdoutW.Close()
|
||||
stderrW.Close()
|
||||
os.RemoveAll(tempPath)
|
||||
close(dr.waitCh)
|
||||
}()
|
||||
|
||||
return dr, nil
|
||||
// blocks return until yt-dlp is downloading or has errored
|
||||
ytErrCh := make(chan error)
|
||||
go func() {
|
||||
stderrLineScanner := bufio.NewScanner(stderrR)
|
||||
for stderrLineScanner.Scan() {
|
||||
const downloadPrefix = "[download]"
|
||||
const errorPrefix = "ERROR: "
|
||||
line := stderrLineScanner.Text()
|
||||
if strings.HasPrefix(line, downloadPrefix) {
|
||||
break
|
||||
} else if strings.HasPrefix(line, errorPrefix) {
|
||||
ytErrCh <- errors.New(line[len(errorPrefix):])
|
||||
return
|
||||
}
|
||||
}
|
||||
ytErrCh <- nil
|
||||
_, _ = io.Copy(io.Discard, stderrR)
|
||||
}()
|
||||
|
||||
return dr, <-ytErrCh
|
||||
}
|
||||
|
||||
func (dr *DownloadResult) Read(p []byte) (n int, err error) {
|
||||
|
@ -7,9 +7,12 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@ -18,14 +21,12 @@ import (
|
||||
"github.com/wader/osleaktest"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// we're using yt-dlp at the moment
|
||||
goutubedl.Path = "yt-dlp"
|
||||
}
|
||||
|
||||
const testVideoRawURL = "https://www.youtube.com/watch?v=C0DPdy98e4c"
|
||||
const playlistRawURL = "https://soundcloud.com/mattheis/sets/kindred-phenomena"
|
||||
const subtitlesTestVideoRawURL = "https://www.youtube.com/watch?v=QRS8MkLhQmM"
|
||||
const (
|
||||
testVideoRawURL = "https://www.youtube.com/watch?v=C0DPdy98e4c"
|
||||
playlistRawURL = "https://soundcloud.com/mattheis/sets/kindred-phenomena"
|
||||
channelRawURL = "https://www.youtube.com/channel/UCHDm-DKoMyJxKVgwGmuTaQA"
|
||||
subtitlesTestVideoRawURL = "https://www.youtube.com/watch?v=QRS8MkLhQmM"
|
||||
)
|
||||
|
||||
func leakChecks(t *testing.T) func() {
|
||||
leakFn := leaktest.Check(t)
|
||||
@ -99,13 +100,45 @@ func TestDownload(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadWithoutInfo(t *testing.T) {
|
||||
defer leakChecks(t)()
|
||||
|
||||
stderrBuf := &bytes.Buffer{}
|
||||
dr, err := goutubedl.Download(context.Background(), testVideoRawURL, goutubedl.Options{
|
||||
StderrFn: func(cmd *exec.Cmd) io.Writer {
|
||||
return stderrBuf
|
||||
},
|
||||
}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
downloadBuf := &bytes.Buffer{}
|
||||
n, err := io.Copy(downloadBuf, dr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dr.Close()
|
||||
|
||||
if n != int64(downloadBuf.Len()) {
|
||||
t.Errorf("copy n not equal to download buffer: %d!=%d", n, downloadBuf.Len())
|
||||
}
|
||||
|
||||
if n < 10000 {
|
||||
t.Errorf("should have copied at least 10000 bytes: %d", n)
|
||||
}
|
||||
|
||||
if !strings.Contains(stderrBuf.String(), "Destination") {
|
||||
t.Errorf("did not find expected log message on stderr: %q", stderrBuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseInfo(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
url string
|
||||
expectedTitle string
|
||||
}{
|
||||
{"https://soundcloud.com/avalonemerson/avalon-emerson-live-at-printworks-london-march-2017", "Avalon Emerson Live at Printworks London 2017"},
|
||||
{"https://www.infoq.com/presentations/Simple-Made-Easy", "Simple Made Easy"},
|
||||
{"https://www.infoq.com/presentations/Simple-Made-Easy", "Simple Made Easy - InfoQ"},
|
||||
{"https://www.youtube.com/watch?v=uVYWQJ5BB_w", "A Radiolab Producer on the Making of a Podcast"},
|
||||
} {
|
||||
t.Run(c.url, func(t *testing.T) {
|
||||
@ -187,7 +220,39 @@ func TestPlaylist(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTestUnsupportedURL(t *testing.T) {
|
||||
func TestChannel(t *testing.T) {
|
||||
defer leakChecks(t)()
|
||||
|
||||
ydlResult, ydlResultErr := goutubedl.New(
|
||||
context.Background(),
|
||||
channelRawURL,
|
||||
goutubedl.Options{
|
||||
Type: goutubedl.TypeChannel,
|
||||
DownloadThumbnail: false,
|
||||
},
|
||||
)
|
||||
|
||||
if ydlResultErr != nil {
|
||||
t.Errorf("failed to download: %s", ydlResultErr)
|
||||
}
|
||||
|
||||
expectedTitle := "Simon Yapp"
|
||||
if ydlResult.Info.Title != expectedTitle {
|
||||
t.Errorf("expected title %q got %q", expectedTitle, ydlResult.Info.Title)
|
||||
}
|
||||
|
||||
expectedEntries := 5
|
||||
if len(ydlResult.Info.Entries) != expectedEntries {
|
||||
t.Errorf("expected %d entries got %d", expectedEntries, len(ydlResult.Info.Entries))
|
||||
}
|
||||
|
||||
expectedTitleOne := "#RNLI Shoreham #LifeBoat demo of launch."
|
||||
if ydlResult.Info.Entries[0].Title != expectedTitleOne {
|
||||
t.Errorf("expected title %q got %q", expectedTitleOne, ydlResult.Info.Entries[0].Title)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsupportedURL(t *testing.T) {
|
||||
defer leaktest.Check(t)()
|
||||
|
||||
_, ydlResultErr := goutubedl.New(context.Background(), "https://www.google.com", goutubedl.Options{})
|
||||
@ -253,6 +318,73 @@ func TestSubtitles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadSections(t *testing.T) {
|
||||
defer leakChecks(t)()
|
||||
|
||||
fileName := "durationTestingFile"
|
||||
duration := 5
|
||||
|
||||
cmd := exec.Command("ffmpeg", "-version")
|
||||
_, err := cmd.Output()
|
||||
if err != nil {
|
||||
t.Errorf("failed to check ffmpeg installed: %s", err)
|
||||
}
|
||||
|
||||
ydlResult, ydlResultErr := goutubedl.New(
|
||||
context.Background(),
|
||||
"https://www.youtube.com/watch?v=OyuL5biOQ94",
|
||||
goutubedl.Options{
|
||||
DownloadSections: fmt.Sprintf("*0:0-0:%d", duration),
|
||||
})
|
||||
|
||||
if ydlResult.Options.DownloadSections != "*0:0-0:5" {
|
||||
t.Errorf("failed to setup --download-sections")
|
||||
}
|
||||
if ydlResultErr != nil {
|
||||
t.Errorf("failed to download: %s", ydlResultErr)
|
||||
}
|
||||
dr, err := ydlResult.Download(context.Background(), "best")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f, err := os.Create(fileName)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
_, err = io.Copy(f, dr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cmd = exec.Command("ffprobe", "-v", "quiet", "-show_entries", "format=duration", fileName)
|
||||
stdout, err := cmd.Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var gotDurationString string
|
||||
output := string(stdout)
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
if strings.Contains(line, "duration") {
|
||||
if d, found := strings.CutPrefix(line, "duration="); found {
|
||||
gotDurationString = d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
gotDuration, err := strconv.ParseFloat(gotDurationString, 32)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seconds := int(gotDuration)
|
||||
if seconds != duration {
|
||||
t.Fatalf("did not get expected duration of %d, but got %d", duration, seconds)
|
||||
}
|
||||
dr.Close()
|
||||
}
|
||||
|
||||
func TestErrorNotAPlaylist(t *testing.T) {
|
||||
defer leakChecks(t)()
|
||||
|
||||
@ -273,6 +405,176 @@ func TestErrorNotASingleEntry(t *testing.T) {
|
||||
DownloadThumbnail: false,
|
||||
})
|
||||
if ydlResultErr != goutubedl.ErrNotASingleEntry {
|
||||
t.Errorf("expected is single entry error, got %s", ydlResultErr)
|
||||
t.Fatalf("expected is single entry error, got %s", ydlResultErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOptionDownloader(t *testing.T) {
|
||||
defer leakChecks(t)()
|
||||
|
||||
ydlResult, ydlResultErr := goutubedl.New(
|
||||
context.Background(),
|
||||
testVideoRawURL,
|
||||
goutubedl.Options{
|
||||
Downloader: "ffmpeg",
|
||||
})
|
||||
|
||||
if ydlResultErr != nil {
|
||||
t.Fatalf("failed to download: %s", ydlResultErr)
|
||||
}
|
||||
dr, err := ydlResult.Download(context.Background(), ydlResult.Info.Formats[0].FormatID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
downloadBuf := &bytes.Buffer{}
|
||||
_, err = io.Copy(downloadBuf, dr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dr.Close()
|
||||
}
|
||||
|
||||
func TestInvalidOptionTypeField(t *testing.T) {
|
||||
defer leakChecks(t)()
|
||||
|
||||
_, err := goutubedl.New(context.Background(), playlistRawURL, goutubedl.Options{
|
||||
Type: 42,
|
||||
})
|
||||
if err == nil {
|
||||
t.Error("should have failed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadPlaylistEntry(t *testing.T) {
|
||||
defer leakChecks(t)()
|
||||
// Download file by specifying the playlist index
|
||||
stderrBuf := &bytes.Buffer{}
|
||||
r, err := goutubedl.New(context.Background(), playlistRawURL, goutubedl.Options{
|
||||
StderrFn: func(cmd *exec.Cmd) io.Writer {
|
||||
return stderrBuf
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
expectedTitle := "Kindred Phenomena"
|
||||
if r.Info.Title != expectedTitle {
|
||||
t.Errorf("expected title %q got %q", expectedTitle, r.Info.Title)
|
||||
}
|
||||
|
||||
expectedEntries := 8
|
||||
if len(r.Info.Entries) != expectedEntries {
|
||||
t.Errorf("expected %d entries got %d", expectedEntries, len(r.Info.Entries))
|
||||
}
|
||||
|
||||
expectedTitleOne := "B1 Mattheis - Ben M"
|
||||
playlistIndex := 2
|
||||
if r.Info.Entries[playlistIndex].Title != expectedTitleOne {
|
||||
t.Errorf("expected title %q got %q", expectedTitleOne, r.Info.Entries[playlistIndex].Title)
|
||||
}
|
||||
|
||||
dr, err := r.DownloadWithOptions(context.Background(), goutubedl.DownloadOptions{
|
||||
PlaylistIndex: int(r.Info.Entries[playlistIndex].PlaylistIndex),
|
||||
Filter: r.Info.Entries[playlistIndex].Formats[0].FormatID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
playlistBuf := &bytes.Buffer{}
|
||||
n, err := io.Copy(playlistBuf, dr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dr.Close()
|
||||
|
||||
if n != int64(playlistBuf.Len()) {
|
||||
t.Errorf("copy n not equal to download buffer: %d!=%d", n, playlistBuf.Len())
|
||||
}
|
||||
|
||||
if n < 10000 {
|
||||
t.Errorf("should have copied at least 10000 bytes: %d", n)
|
||||
}
|
||||
|
||||
if !strings.Contains(stderrBuf.String(), "Destination") {
|
||||
t.Errorf("did not find expected log message on stderr: %q", stderrBuf.String())
|
||||
}
|
||||
|
||||
// Download the same file but with the direct link
|
||||
url := "https://soundcloud.com/mattheis/b1-mattheis-ben-m"
|
||||
stderrBuf = &bytes.Buffer{}
|
||||
r, err = goutubedl.New(context.Background(), url, goutubedl.Options{
|
||||
StderrFn: func(cmd *exec.Cmd) io.Writer {
|
||||
return stderrBuf
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if r.Info.Title != expectedTitleOne {
|
||||
t.Errorf("expected title %q got %q", expectedTitleOne, r.Info.Title)
|
||||
}
|
||||
|
||||
expectedEntries = 0
|
||||
if len(r.Info.Entries) != expectedEntries {
|
||||
t.Errorf("expected %d entries got %d", expectedEntries, len(r.Info.Entries))
|
||||
}
|
||||
|
||||
dr, err = r.Download(context.Background(), r.Info.Formats[0].FormatID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
directLinkBuf := &bytes.Buffer{}
|
||||
n, err = io.Copy(directLinkBuf, dr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dr.Close()
|
||||
|
||||
if n != int64(directLinkBuf.Len()) {
|
||||
t.Errorf("copy n not equal to download buffer: %d!=%d", n, directLinkBuf.Len())
|
||||
}
|
||||
|
||||
if n < 10000 {
|
||||
t.Errorf("should have copied at least 10000 bytes: %d", n)
|
||||
}
|
||||
|
||||
if !strings.Contains(stderrBuf.String(), "Destination") {
|
||||
t.Errorf("did not find expected log message on stderr: %q", stderrBuf.String())
|
||||
}
|
||||
|
||||
if directLinkBuf.Len() != playlistBuf.Len() {
|
||||
t.Errorf("not the same content size between the playlist index entry and the direct link entry: %d != %d", playlistBuf.Len(), directLinkBuf.Len())
|
||||
}
|
||||
|
||||
if !bytes.Equal(directLinkBuf.Bytes(), playlistBuf.Bytes()) {
|
||||
t.Error("not the same content between the playlist index entry and the direct link entry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatDownloadError(t *testing.T) {
|
||||
defer leaktest.Check(t)()
|
||||
|
||||
ydl, ydlErr := goutubedl.New(
|
||||
context.Background(),
|
||||
"https://www.reddit.com/r/newsbabes/s/92rflI0EB0",
|
||||
goutubedl.Options{},
|
||||
)
|
||||
|
||||
if ydlErr != nil {
|
||||
// reddit seems to not like github action hosts
|
||||
if strings.Contains(ydlErr.Error(), "HTTPError 403: Blocked") {
|
||||
t.Skip()
|
||||
}
|
||||
t.Error(ydlErr)
|
||||
}
|
||||
|
||||
// no pre-muxed audio/video format available
|
||||
_, ytDlErr := ydl.Download(context.Background(), "best")
|
||||
expectedErr := "Requested format is not available"
|
||||
if ydlErr != nil && !strings.Contains(ytDlErr.Error(), expectedErr) {
|
||||
t.Errorf("expected error prefix %q got %q", expectedErr, ytDlErr.Error())
|
||||
|
||||
}
|
||||
}
|
||||
|
Reference in New Issue
Block a user