From 06a1d2ac41f4bd0992865eb10baefb6742222495 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 11 Sep 2021 22:47:12 -0700 Subject: Rename fetch_videojs* to fetch_player* --- scripts/fetch-player-dependencies.cr | 157 +++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 scripts/fetch-player-dependencies.cr (limited to 'scripts/fetch-player-dependencies.cr') diff --git a/scripts/fetch-player-dependencies.cr b/scripts/fetch-player-dependencies.cr new file mode 100644 index 00000000..ba9722f8 --- /dev/null +++ b/scripts/fetch-player-dependencies.cr @@ -0,0 +1,157 @@ +require "http" +require "yaml" +require "digest/sha1" +require "option_parser" +require "colorize" + +# Taken from https://crystal-lang.org/api/1.1.1/OptionParser.html +minified = false +OptionParser.parse do |parser| + parser.banner = "Usage: Fetch VideoJS dependencies [arguments]" + parser.on("-m", "--minified", "Use minified versions of VideoJS dependencies (performance and bandwidth benefit)") { minified = true } + + parser.on("-h", "--help", "Show this help") do + puts parser + exit + end + + parser.invalid_option do |flag| + STDERR.puts "ERROR: #{flag} is not a valid option." + STDERR.puts parser + exit(1) + end +end + +required_dependencies = File.open("videojs-dependencies.yml") do |file| + YAML.parse(file).as_h +end + +def update_versions_yaml(required_dependencies, minified, dep_name) + File.open("assets/videojs/#{dep_name}/versions.yml", "w") do |io| + YAML.build(io) do |builder| + builder.mapping do + # Versions + builder.scalar "version" + builder.scalar "#{required_dependencies[dep_name]["version"]}" + + builder.scalar "minified" + builder.scalar minified + end + end + end +end + +# The first step is to check which dependencies we'll need to install. +# If the version we have requested in `videojs-dependencies.yml` is the +# same as what we've installed, we shouldn't do anything. Likewise, if it's +# different or the requested dependency just isn't present, then it needs to be +# installed. + +# Since we can't know when videojs-youtube-annotations is updated, we'll just always fetch +# a new copy each time. +dependencies_to_install = [] of String + +required_dependencies.keys.each do |dep| + dep = dep.to_s + path = "assets/videojs/#{dep}" + # Check for missing dependencies + if !Dir.exists?(path) + Dir.mkdir(path) + + update_versions_yaml(required_dependencies, minified, dep) + dependencies_to_install << dep + else + config = File.open("#{path}/versions.yml") do |file| + YAML.parse(file).as_h + end + + if config["version"].as_s != required_dependencies[dep]["version"].as_s || config["minified"].as_bool != minified + `rm -rf #{path}/*` + dependencies_to_install << dep + update_versions_yaml(required_dependencies, minified, dep) + end + end +end + +# Now we begin the fun part of installing the dependencies. +# But first we'll setup a temp directory to store the plugins +tmp_dir_path = "#{Dir.tempdir}/invidious-videojs-dep-install" +Dir.mkdir(tmp_dir_path) if !Dir.exists? tmp_dir_path + +channel = Channel(String).new + +dependencies_to_install.each do |dep| + spawn do + dep_name = dep + download_path = "#{tmp_dir_path}/#{dep}" + dest_path = "assets/videojs/#{dep}" + + HTTP::Client.get("https://registry.npmjs.org/#{dep}/-/#{dep}-#{required_dependencies[dep]["version"]}.tgz") do |response| + Dir.mkdir(download_path) + data = response.body_io.gets_to_end + File.write("#{download_path}/package.tgz", data) + + if Digest::SHA1.hexdigest(data) != required_dependencies[dep]["shasum"] + raise Exception.new("Checksum for '#{dep}' failed") + end + end + + # Unless we install an external dependency, crystal provides no way of extracting a tarball. + # Thus we'll go ahead and call a system command. + args = Process.parse_arguments("-zxvf '#{download_path}/package.tgz' -C '#{download_path}'") + process = Process.new("tar", args: args) + process.wait.success? # => true + + # Would use File.rename in the following steps but for some reason it just doesn't work here. + # Video.js itself is structured slightly differently + dep = "video" if dep == "video.js" + + # This dep nests everything under an additional JS or CSS folder + if dep == "silvermine-videojs-quality-selector" + js_path = "js/" + + # It also stores their quality selector as `quality-selector.css` + `mv #{download_path}/package/dist/css/quality-selector.css #{dest_path}/quality-selector.css` + else + js_path = "" + end + + # Would use File.rename but for some reason it just doesn't work here. + if minified && File.exists?("#{download_path}/package/dist/#{dep}.min.js") + `mv #{download_path}/package/dist/#{js_path}#{dep}.min.js #{dest_path}/#{dep}.js` + else + `mv #{download_path}/package/dist/#{js_path}#{dep}.js #{dest_path}/#{dep}.js` + end + + # Fetch CSS which isn't guaranteed to exist + # + # Also, video JS changes structure here once again... + dep = "video-js" if dep == "video" + + # VideoJS marker uses a dot on the CSS files. + dep = "videojs.markers" if dep == "videojs-markers" + + if File.exists?("#{download_path}/package/dist/#{dep}.css") + if minified && File.exists?("#{tmp_dir_path}/#{dep}/package/dist/#{dep}.min.css") + `mv #{download_path}/package/dist/#{dep}.min.css #{dest_path}/#{dep}.css` + else + `mv #{download_path}/package/dist/#{dep}.css #{dest_path}/#{dep}.css` + end + end + + channel.send(dep_name) + end +end + +if dependencies_to_install.empty? + puts "#{"VideoJS".colorize(:blue)} #{"dependencies".colorize(:green)} are satisfied" +else + puts "#{"Resolving".colorize(:green)} #{"VideoJS".colorize(:blue)} dependencies" + dependencies_to_install.size.times do + result = channel.receive + puts "#{"Fetched".colorize(:green)} #{result.colorize(:blue)}" + end +end + +# Cleanup +`rm -rf #{tmp_dir_path}` -- cgit v1.2.3 From 1af1474d04c484150cfa8e22297aced25032c984 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 11 Sep 2021 22:47:51 -0700 Subject: Change "VideoJS" in put reports to "Player" --- scripts/fetch-player-dependencies.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts/fetch-player-dependencies.cr') diff --git a/scripts/fetch-player-dependencies.cr b/scripts/fetch-player-dependencies.cr index ba9722f8..2af8cd09 100644 --- a/scripts/fetch-player-dependencies.cr +++ b/scripts/fetch-player-dependencies.cr @@ -144,9 +144,9 @@ dependencies_to_install.each do |dep| end if dependencies_to_install.empty? - puts "#{"VideoJS".colorize(:blue)} #{"dependencies".colorize(:green)} are satisfied" + puts "#{"Player".colorize(:blue)} #{"dependencies".colorize(:green)} are satisfied" else - puts "#{"Resolving".colorize(:green)} #{"VideoJS".colorize(:blue)} dependencies" + puts "#{"Resolving".colorize(:green)} #{"player".colorize(:blue)} dependencies" dependencies_to_install.size.times do result = channel.receive puts "#{"Fetched".colorize(:green)} #{result.colorize(:blue)}" -- cgit v1.2.3 From b6670a7e3d8f5319810a99766296269ee458d73f Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 12 Sep 2021 00:09:26 -0700 Subject: Fix minified silvermine-...quality* fetching --- scripts/fetch-player-dependencies.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/fetch-player-dependencies.cr') diff --git a/scripts/fetch-player-dependencies.cr b/scripts/fetch-player-dependencies.cr index 2af8cd09..c24b6e0f 100644 --- a/scripts/fetch-player-dependencies.cr +++ b/scripts/fetch-player-dependencies.cr @@ -117,7 +117,7 @@ dependencies_to_install.each do |dep| end # Would use File.rename but for some reason it just doesn't work here. - if minified && File.exists?("#{download_path}/package/dist/#{dep}.min.js") + if minified && File.exists?("#{download_path}/package/dist/#{js_path}#{dep}.min.js") `mv #{download_path}/package/dist/#{js_path}#{dep}.min.js #{dest_path}/#{dep}.js` else `mv #{download_path}/package/dist/#{js_path}#{dep}.js #{dest_path}/#{dep}.js` -- cgit v1.2.3 From 4e629ca858ee21acebe458342311afab3ecd4155 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 18 Sep 2021 14:42:41 -0700 Subject: Use shell command `sha1sum` for checksum Crystal doesn't support OpenSSL3, the version Alpine uses. See https://github.com/iv-org/invidious/pull/2397#issuecomment-922375908 --- scripts/fetch-player-dependencies.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts/fetch-player-dependencies.cr') diff --git a/scripts/fetch-player-dependencies.cr b/scripts/fetch-player-dependencies.cr index c24b6e0f..ba4c97aa 100644 --- a/scripts/fetch-player-dependencies.cr +++ b/scripts/fetch-player-dependencies.cr @@ -91,7 +91,8 @@ dependencies_to_install.each do |dep| data = response.body_io.gets_to_end File.write("#{download_path}/package.tgz", data) - if Digest::SHA1.hexdigest(data) != required_dependencies[dep]["shasum"] + # https://github.com/iv-org/invidious/pull/2397#issuecomment-922375908 + if `sha1sum #{download_path}/package.tgz`.split(" ")[0] != required_dependencies[dep]["shasum"] raise Exception.new("Checksum for '#{dep}' failed") end end -- cgit v1.2.3 From 9be8263f2681f97c4e2bbf1fc661e120bce07728 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 2 Oct 2021 05:52:42 -0700 Subject: Use command literal for extraction --- scripts/fetch-player-dependencies.cr | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'scripts/fetch-player-dependencies.cr') diff --git a/scripts/fetch-player-dependencies.cr b/scripts/fetch-player-dependencies.cr index ba4c97aa..c141d80a 100644 --- a/scripts/fetch-player-dependencies.cr +++ b/scripts/fetch-player-dependencies.cr @@ -99,9 +99,8 @@ dependencies_to_install.each do |dep| # Unless we install an external dependency, crystal provides no way of extracting a tarball. # Thus we'll go ahead and call a system command. - args = Process.parse_arguments("-zxvf '#{download_path}/package.tgz' -C '#{download_path}'") - process = Process.new("tar", args: args) - process.wait.success? # => true + `tar -zxf '#{download_path}/package.tgz' -C '#{download_path}'"` + raise "Extraction for #{dep} failed" if !$?.success? # Would use File.rename in the following steps but for some reason it just doesn't work here. # Video.js itself is structured slightly differently -- cgit v1.2.3 From 1f1e14fba5ded3484745390676bf8937f297d282 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 2 Oct 2021 06:07:48 -0700 Subject: Propagate exceptions from fiber --- scripts/fetch-player-dependencies.cr | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'scripts/fetch-player-dependencies.cr') diff --git a/scripts/fetch-player-dependencies.cr b/scripts/fetch-player-dependencies.cr index c141d80a..adfe57ba 100644 --- a/scripts/fetch-player-dependencies.cr +++ b/scripts/fetch-player-dependencies.cr @@ -78,7 +78,7 @@ end tmp_dir_path = "#{Dir.tempdir}/invidious-videojs-dep-install" Dir.mkdir(tmp_dir_path) if !Dir.exists? tmp_dir_path -channel = Channel(String).new +channel = Channel(String | Exception).new dependencies_to_install.each do |dep| spawn do @@ -99,7 +99,7 @@ dependencies_to_install.each do |dep| # Unless we install an external dependency, crystal provides no way of extracting a tarball. # Thus we'll go ahead and call a system command. - `tar -zxf '#{download_path}/package.tgz' -C '#{download_path}'"` + `tar -vzxf '#{download_path}/package.tgz' -C '#{download_path}'` raise "Extraction for #{dep} failed" if !$?.success? # Would use File.rename in the following steps but for some reason it just doesn't work here. @@ -140,6 +140,8 @@ dependencies_to_install.each do |dep| end channel.send(dep_name) + rescue ex + channel.send(ex) end end @@ -149,6 +151,11 @@ else puts "#{"Resolving".colorize(:green)} #{"player".colorize(:blue)} dependencies" dependencies_to_install.size.times do result = channel.receive + + if result.is_a? Exception + raise result + end + puts "#{"Fetched".colorize(:green)} #{result.colorize(:blue)}" end end -- cgit v1.2.3 From 0c7726d4e1a66e1a1d1761f81951c87adf057fe6 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 2 Oct 2021 06:10:57 -0700 Subject: Update/create versions.yml after dep installation --- scripts/fetch-player-dependencies.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts/fetch-player-dependencies.cr') diff --git a/scripts/fetch-player-dependencies.cr b/scripts/fetch-player-dependencies.cr index adfe57ba..ed658b51 100644 --- a/scripts/fetch-player-dependencies.cr +++ b/scripts/fetch-player-dependencies.cr @@ -57,8 +57,6 @@ required_dependencies.keys.each do |dep| # Check for missing dependencies if !Dir.exists?(path) Dir.mkdir(path) - - update_versions_yaml(required_dependencies, minified, dep) dependencies_to_install << dep else config = File.open("#{path}/versions.yml") do |file| @@ -66,9 +64,8 @@ required_dependencies.keys.each do |dep| end if config["version"].as_s != required_dependencies[dep]["version"].as_s || config["minified"].as_bool != minified - `rm -rf #{path}/*` + `rm -rf #{path}/*.js #{path}/*.css` dependencies_to_install << dep - update_versions_yaml(required_dependencies, minified, dep) end end end @@ -139,6 +136,9 @@ dependencies_to_install.each do |dep| end end + # Update/create versions file for the dependency + update_versions_yaml(required_dependencies, minified, dep_name) + channel.send(dep_name) rescue ex channel.send(ex) -- cgit v1.2.3