diff --git a/README.md b/README.md index 97c9a2f8..166e050c 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,8 @@ Onion links: - kgg2m7yk5aybusll.onion - axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid.onion +[Alternative Invidious instances](https://github.com/omarroth/invidious/wiki/Invidious-Instances) + ## Installation ### Docker: @@ -57,71 +59,94 @@ $ docker volume rm invidious_postgresdata $ docker-compose build ``` -### Arch Linux: +### Linux: + +#### Install dependencies ```bash -# Install dependencies -$ sudo pacman -S shards crystal imagemagick librsvg +# Arch Linux +$ sudo pacman -S shards crystal imagemagick librsvg postgresql -# Setup PostgresSQL -$ sudo systemctl enable postgresql -$ sudo systemctl start postgresql -$ sudo -i -u postgres -$ createuser -s YOUR_USER_NAME -$ createdb YOUR_USER_NAME -$ exit - -# Setup Invidious -$ git clone https://github.com/omarroth/invidious -$ cd invidious -$ ./setup.sh -$ shards -$ crystal build src/invidious.cr --release -``` - -### On Ubuntu: - -```bash -# Install dependencies +# Ubuntu or Debian $ curl -sSL https://dist.crystal-lang.org/apt/setup.sh | sudo bash $ sudo apt update $ sudo apt install crystal libssl-dev libxml2-dev libyaml-dev libgmp-dev libreadline-dev librsvg2-dev postgresql imagemagick libsqlite3-dev +``` -# Setup PostgreSQL +#### Add invidious user and clone repository + +```bash +$ useradd -m invidious +$ sudo -i -u invidious +$ git clone https://github.com/omarroth/invidious +$ exit +``` + +#### Setup PostgresSQL + +```bash $ sudo systemctl enable postgresql $ sudo systemctl start postgresql $ sudo -i -u postgres -$ createuser -s YOUR_USER_NAME_HERE -$ createdb YOUR_USER_NAME_HERE +$ psql -c "CREATE USER kemal WITH PASSWORD 'kemal';" +$ createdb -O kemal invidious +$ psql invidious < /home/invidious/invidious/config/sql/channels.sql +$ psql invidious < /home/invidious/invidious/config/sql/videos.sql +$ psql invidious < /home/invidious/invidious/config/sql/channel_videos.sql +$ psql invidious < /home/invidious/invidious/config/sql/users.sql +$ psql invidious < /home/invidious/invidious/config/sql/nonces.sql $ exit - -# Setup Invidious -$ git clone https://github.com/omarroth/invidious -$ cd invidious -$ ./setup.sh -$ shards -$ crystal build src/invidious.cr --release ``` -### On OSX: +#### Setup Invidious + +```bash +$ sudo -i -u invidious +$ cd invidious +$ shards +$ crystal build src/invidious.cr --release +# test compiled binary +$ ./invidious # stop with ctrl c +$ exit +``` + +#### systemd service +```bash +$ sudo cp invidious.service /etc/systemd/system/invidious.service +$ sudo systemctl enable invidious.service +$ sudo systemctl start invidious.service +``` + +### OSX: ```bash # Install dependencies $ brew update $ brew install shards crystal-lang postgres imagemagick librsvg -# Setup Invidious +# Clone repository and setup postgres database $ git clone https://github.com/omarroth/invidious $ cd invidious -$ ./setup.sh +$ brew services start postgresql +$ psql -c "CREATE ROLE kemal WITH LOGIN PASSWORD 'kemal';" +$ createdb invidious -U kemal +$ psql invidious < config/sql/channels.sql +$ psql invidious < config/sql/videos.sql +$ psql invidious < config/sql/channel_videos.sql +$ psql invidious < config/sql/users.sql +$ psql invidious < config/sql/nonces.sql + +# Setup Invidious $ shards $ crystal build src/invidious.cr --release ``` +## Update Invidious +You can find information about how to update in the wiki: [Update Invidious](https://github.com/omarroth/invidious/wiki/Update-Invidious) + ## Usage: ```bash -$ crystal build src/invidious.cr --release $ ./invidious -h Usage: invidious [arguments] -b HOST, --bind HOST Host to bind (defaults to 0.0.0.0) @@ -131,13 +156,14 @@ Usage: invidious [arguments] --ssl-cert-file FILE SSL certificate file -h, --help Shows this help -t THREADS, --crawl-threads=THREADS - Number of threads for crawling (default: 1) + Number of threads for crawling YouTube (default: 0) -c THREADS, --channel-threads=THREADS Number of threads for refreshing channels (default: 1) -f THREADS, --feed-threads=THREADS Number of threads for refreshing feeds (default: 1) -v THREADS, --video-threads=THREADS - Number of threads for refreshing videos (default: 1) + Number of threads for refreshing videos (default: 0) + -o OUTPUT, --output=OUTPUT Redirect output (default: STDOUT) ``` Or for development: @@ -147,15 +173,8 @@ $ curl -fsSLo- https://raw.githubusercontent.com/samueleaton/sentry/master/insta $ ./sentry ``` -## Optional - -Create a systemd service to run Invidious in background. Edit `invidious.service` to change your installation path and log location. Than copy and enable the systemd service. - -``` -$ sudo cp invidious.service /etc/systemd/system/invidious.service -$ sudo systemctl enable invidious.service -$ sudo systemctl start invidious.service -``` +## Documentation +[Documentation](https://github.com/omarroth/invidious/wiki) can be found in the wiki. ## Extensions diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres index 3186e050..720bdff8 100644 --- a/docker/Dockerfile.postgres +++ b/docker/Dockerfile.postgres @@ -2,7 +2,6 @@ FROM postgres:10 ENV POSTGRES_USER postgres -ADD ./setup.sh /setup.sh ADD ./config/sql /config/sql ADD ./docker/entrypoint.postgres.sh /entrypoint.sh diff --git a/docker/entrypoint.postgres.sh b/docker/entrypoint.postgres.sh index 2f3ae65f..9a258dd6 100755 --- a/docker/entrypoint.postgres.sh +++ b/docker/entrypoint.postgres.sh @@ -10,7 +10,14 @@ if [ ! -f /var/lib/postgresql/data/setupFinished ]; then sleep 5 done >&2 echo "### importing table schemas" - su postgres -c "/setup.sh" && touch /var/lib/postgresql/data/setupFinished + su postgres -c 'createdb invidious' + su postgres -c 'psql -c "CREATE USER kemal WITH PASSWORD '"'kemal'"'"' + su postgres -c 'psql invidious < config/sql/channels.sql' + su postgres -c 'psql invidious < config/sql/videos.sql' + su postgres -c 'psql invidious < config/sql/channel_videos.sql' + su postgres -c 'psql invidious < config/sql/users.sql' + su postgres -c 'psql invidious < config/sql/nonces.sql' + touch /var/lib/postgresql/data/setupFinished echo "### invidious database setup finished" exit fi diff --git a/invidious.service b/invidious.service index 556b862f..6ef12ccf 100644 --- a/invidious.service +++ b/invidious.service @@ -7,19 +7,13 @@ After=network.target RestartSec=2s Type=simple -# set user and group User=invidious Group=invidious -# configure location -WorkingDirectory=/home/invidious -ExecStart=/home/invidious/invidious/invidious +WorkingDirectory=/home/invidious/invidious +ExecStart=/home/invidious/invidious/invidious -o invidious.log Restart=always -# default log output is syslog, to disable log enable both -#StandardOutput=null -#StandardError=null - [Install] WantedBy=multi-user.target diff --git a/setup.sh b/setup.sh deleted file mode 100755 index 7b708897..00000000 --- a/setup.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -createdb invidious -#createuser kemal -psql -c "CREATE USER kemal WITH PASSWORD 'kemal';" -psql invidious < config/sql/channels.sql -psql invidious < config/sql/videos.sql -psql invidious < config/sql/channel_videos.sql -psql invidious < config/sql/users.sql -psql invidious < config/sql/nonces.sql diff --git a/src/invidious.cr b/src/invidious.cr index 7f77e317..f3404d23 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -108,17 +108,17 @@ LOCALES = { crawl_threads.times do spawn do - crawl_videos(PG_DB) + crawl_videos(PG_DB, logger) end end -refresh_channels(PG_DB, channel_threads, CONFIG.full_refresh) +refresh_channels(PG_DB, logger, channel_threads, CONFIG.full_refresh) -refresh_feeds(PG_DB, feed_threads) +refresh_feeds(PG_DB, logger, feed_threads) video_threads.times do |i| spawn do - refresh_videos(PG_DB) + refresh_videos(PG_DB, logger) end end @@ -126,6 +126,8 @@ top_videos = [] of Video spawn do pull_top_videos(CONFIG, PG_DB) do |videos| top_videos = videos + sleep 1.minutes + Fiber.yield end end @@ -133,6 +135,8 @@ popular_videos = [] of ChannelVideo spawn do pull_popular_videos(PG_DB) do |videos| popular_videos = videos + sleep 1.minutes + Fiber.yield end end @@ -3644,9 +3648,23 @@ get "/videoplayback" do |env| host = "https://r#{fvip}---#{mn}.googlevideo.com" url = "/videoplayback?#{query_params.to_s}" + headers = env.request.headers + headers.delete("Host") + headers.delete("Cookie") + headers.delete("User-Agent") + headers.delete("Referer") + region = query_params["region"]? - client = make_client(URI.parse(host), proxies, region) - response = client.head(url) + + response = HTTP::Client::Response.new(403) + loop do + begin + client = make_client(URI.parse(host), proxies, region) + response = client.head(url, headers) + break + rescue ex + end + end if response.headers["Location"]? url = URI.parse(response.headers["Location"]) @@ -3664,12 +3682,6 @@ get "/videoplayback" do |env| halt env, status_code: 403 end - headers = env.request.headers - headers.delete("Host") - headers.delete("Cookie") - headers.delete("User-Agent") - headers.delete("Referer") - client = make_client(URI.parse(host), proxies, region) client.get(url, headers) do |response| env.response.status_code = response.status_code diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr index df02c7fb..f6e2d8fe 100644 --- a/src/invidious/jobs.cr +++ b/src/invidious/jobs.cr @@ -1,4 +1,4 @@ -def crawl_videos(db) +def crawl_videos(db, logger) ids = Deque(String).new random = Random.new @@ -21,7 +21,7 @@ def crawl_videos(db) id = ids[0] video = get_video(id, db) rescue ex - STDOUT << id << " : " << ex.message << "\n" + logger.write("#{id} : #{ex.message}\n") next ensure ids.delete(id) @@ -46,7 +46,7 @@ def crawl_videos(db) end end -def refresh_channels(db, max_threads = 1, full_refresh = false) +def refresh_channels(db, logger, max_threads = 1, full_refresh = false) max_channel = Channel(Int32).new spawn do @@ -73,7 +73,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false) db.exec("UPDATE channels SET updated = $1, author = $2 WHERE id = $3", Time.now, channel.author, id) rescue ex - STDOUT << id << " : " << ex.message << "\n" + logger.write("#{id} : #{ex.message}\n") end active_channel.send(true) @@ -86,7 +86,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false) max_channel.send(max_threads) end -def refresh_videos(db) +def refresh_videos(db, logger) loop do db.query("SELECT id FROM videos ORDER BY updated") do |rs| rs.each do @@ -94,7 +94,7 @@ def refresh_videos(db) id = rs.read(String) video = get_video(id, db) rescue ex - STDOUT << id << " : " << ex.message << "\n" + logger.write("#{id} : #{ex.message}\n") next end end @@ -104,7 +104,7 @@ def refresh_videos(db) end end -def refresh_feeds(db, max_threads = 1) +def refresh_feeds(db, logger, max_threads = 1) max_channel = Channel(Int32).new spawn do @@ -129,7 +129,7 @@ def refresh_feeds(db, max_threads = 1) begin db.exec("REFRESH MATERIALIZED VIEW #{view_name}") rescue ex - STDOUT << "REFRESH " << email << " : " << ex.message << "\n" + logger.write("REFRESH #{email} : #{ex.message}\n") end active_channel.send(true)