Merge branch 'master' into master

README.md

@@ -32,6 +32,8 @@ Onion links:
 - kgg2m7yk5aybusll.onion
 - axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid.onion
 
+[Alternative Invidious instances](https://github.com/omarroth/invidious/wiki/Invidious-Instances)
+
 ## Installation
 
 ### Docker:
@@ -57,71 +59,94 @@ $ docker volume rm invidious_postgresdata
 $ docker-compose build
 ```
 
-### Arch Linux:
+### Linux:
+
+#### Install dependencies
 
 ```bash
-# Install dependencies
-$ sudo pacman -S shards crystal imagemagick librsvg
+# Arch Linux
+$ sudo pacman -S shards crystal imagemagick librsvg postgresql
 
-# Setup PostgresSQL
-$ sudo systemctl enable postgresql
-$ sudo systemctl start postgresql
-$ sudo -i -u postgres
-$ createuser -s YOUR_USER_NAME
-$ createdb      YOUR_USER_NAME
-$ exit
-
-# Setup Invidious
-$ git clone https://github.com/omarroth/invidious
-$ cd invidious
-$ ./setup.sh
-$ shards
-$ crystal build src/invidious.cr --release
-```
-
-### On Ubuntu:
-
-```bash
-# Install dependencies
+# Ubuntu or Debian
 $ curl -sSL https://dist.crystal-lang.org/apt/setup.sh | sudo bash
 $ sudo apt update
 $ sudo apt install crystal libssl-dev libxml2-dev libyaml-dev libgmp-dev libreadline-dev librsvg2-dev postgresql imagemagick libsqlite3-dev
+```
 
-# Setup PostgreSQL
+#### Add invidious user and clone repository
+
+```bash
+$ useradd -m invidious
+$ sudo -i -u invidious
+$ git clone https://github.com/omarroth/invidious
+$ exit
+```
+
+#### Setup PostgresSQL
+
+```bash
 $ sudo systemctl enable postgresql
 $ sudo systemctl start postgresql
 $ sudo -i -u postgres
-$ createuser -s YOUR_USER_NAME_HERE
-$ createdb      YOUR_USER_NAME_HERE
+$ psql -c "CREATE USER kemal WITH PASSWORD 'kemal';"
+$ createdb -O kemal invidious
+$ psql invidious < /home/invidious/invidious/config/sql/channels.sql
+$ psql invidious < /home/invidious/invidious/config/sql/videos.sql
+$ psql invidious < /home/invidious/invidious/config/sql/channel_videos.sql
+$ psql invidious < /home/invidious/invidious/config/sql/users.sql
+$ psql invidious < /home/invidious/invidious/config/sql/nonces.sql
 $ exit
-
-# Setup Invidious
-$ git clone https://github.com/omarroth/invidious
-$ cd invidious
-$ ./setup.sh
-$ shards
-$ crystal build src/invidious.cr --release
 ```
 
-### On OSX:
+#### Setup Invidious
+
+```bash
+$ sudo -i -u invidious
+$ cd invidious
+$ shards
+$ crystal build src/invidious.cr --release
+# test compiled binary
+$ ./invidious # stop with ctrl c
+$ exit
+```
+
+#### systemd service
+```bash
+$ sudo cp invidious.service /etc/systemd/system/invidious.service
+$ sudo systemctl enable invidious.service
+$ sudo systemctl start invidious.service
+```
+
+### OSX:
 
 ```bash
 # Install dependencies
 $ brew update
 $ brew install shards crystal-lang postgres imagemagick librsvg
 
-# Setup Invidious
+# Clone repository and setup postgres database
 $ git clone https://github.com/omarroth/invidious
 $ cd invidious
-$ ./setup.sh
+$ brew services start postgresql
+$ psql -c "CREATE ROLE kemal WITH LOGIN PASSWORD 'kemal';"
+$ createdb invidious -U kemal
+$ psql invidious < config/sql/channels.sql
+$ psql invidious < config/sql/videos.sql
+$ psql invidious < config/sql/channel_videos.sql
+$ psql invidious < config/sql/users.sql
+$ psql invidious < config/sql/nonces.sql
+
+# Setup Invidious
 $ shards
 $ crystal build src/invidious.cr --release
 ```
 
+## Update Invidious
+You can find information about how to update in the wiki: [Update Invidious](https://github.com/omarroth/invidious/wiki/Update-Invidious)
+
 ## Usage:
 
 ```bash
-$ crystal build src/invidious.cr --release
 $ ./invidious -h
 Usage: invidious [arguments]
     -b HOST, --bind HOST             Host to bind (defaults to 0.0.0.0)
@@ -131,13 +156,14 @@ Usage: invidious [arguments]
     --ssl-cert-file FILE             SSL certificate file
     -h, --help                       Shows this help
     -t THREADS, --crawl-threads=THREADS
-                                     Number of threads for crawling (default: 1)
+                                     Number of threads for crawling YouTube (default: 0)
     -c THREADS, --channel-threads=THREADS
                                      Number of threads for refreshing channels (default: 1)
     -f THREADS, --feed-threads=THREADS
                                      Number of threads for refreshing feeds (default: 1)
     -v THREADS, --video-threads=THREADS
-                                     Number of threads for refreshing videos (default: 1)
+                                     Number of threads for refreshing videos (default: 0)
+    -o OUTPUT, --output=OUTPUT       Redirect output (default: STDOUT)
 ```
 
 Or for development:
@@ -147,15 +173,8 @@ $ curl -fsSLo- https://raw.githubusercontent.com/samueleaton/sentry/master/insta
 $ ./sentry
 ```
 
-## Optional
-
-Create a systemd service to run Invidious in background. Edit `invidious.service` to change your installation path and log location. Than copy and enable the systemd service.
-
-```
-$ sudo cp invidious.service /etc/systemd/system/invidious.service
-$ sudo systemctl enable invidious.service
-$ sudo systemctl start invidious.service
-```
+## Documentation
+[Documentation](https://github.com/omarroth/invidious/wiki) can be found in the wiki.
 
 ## Extensions
 

docker/Dockerfile.postgres

@@ -2,7 +2,6 @@ FROM postgres:10
 
 ENV POSTGRES_USER postgres
 
-ADD ./setup.sh /setup.sh
 ADD ./config/sql /config/sql
 ADD ./docker/entrypoint.postgres.sh /entrypoint.sh
 

docker/entrypoint.postgres.sh

@@ -10,7 +10,14 @@ if [ ! -f /var/lib/postgresql/data/setupFinished ]; then
         sleep 5
     done
     >&2 echo "### importing table schemas"
-    su postgres -c "/setup.sh" && touch /var/lib/postgresql/data/setupFinished
+    su postgres -c 'createdb invidious'
+    su postgres -c 'psql -c "CREATE USER kemal WITH PASSWORD '"'kemal'"'"'
+    su postgres -c 'psql invidious < config/sql/channels.sql'
+    su postgres -c 'psql invidious < config/sql/videos.sql'
+    su postgres -c 'psql invidious < config/sql/channel_videos.sql'
+    su postgres -c 'psql invidious < config/sql/users.sql'
+    su postgres -c 'psql invidious < config/sql/nonces.sql'
+    touch /var/lib/postgresql/data/setupFinished
     echo "### invidious database setup finished"
     exit
 fi

invidious.service

@@ -7,19 +7,13 @@ After=network.target
 RestartSec=2s
 Type=simple
 
-# set user and group
 User=invidious
 Group=invidious
 
-# configure location
-WorkingDirectory=/home/invidious
-ExecStart=/home/invidious/invidious/invidious
+WorkingDirectory=/home/invidious/invidious
+ExecStart=/home/invidious/invidious/invidious -o invidious.log
 
 Restart=always
 
-# default log output is syslog, to disable log enable both
-#StandardOutput=null
-#StandardError=null
-
 [Install]
 WantedBy=multi-user.target

setup.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-createdb invidious
-#createuser kemal
-psql -c "CREATE USER kemal WITH PASSWORD 'kemal';"
-psql invidious < config/sql/channels.sql
-psql invidious < config/sql/videos.sql
-psql invidious < config/sql/channel_videos.sql
-psql invidious < config/sql/users.sql
-psql invidious < config/sql/nonces.sql

src/invidious.cr

@@ -108,17 +108,17 @@ LOCALES = {
 
 crawl_threads.times do
   spawn do
-    crawl_videos(PG_DB)
+    crawl_videos(PG_DB, logger)
   end
 end
 
-refresh_channels(PG_DB, channel_threads, CONFIG.full_refresh)
+refresh_channels(PG_DB, logger, channel_threads, CONFIG.full_refresh)
 
-refresh_feeds(PG_DB, feed_threads)
+refresh_feeds(PG_DB, logger, feed_threads)
 
 video_threads.times do |i|
   spawn do
-    refresh_videos(PG_DB)
+    refresh_videos(PG_DB, logger)
   end
 end
 
@@ -126,6 +126,8 @@ top_videos = [] of Video
 spawn do
   pull_top_videos(CONFIG, PG_DB) do |videos|
     top_videos = videos
+    sleep 1.minutes
+    Fiber.yield
   end
 end
 
@@ -133,6 +135,8 @@ popular_videos = [] of ChannelVideo
 spawn do
   pull_popular_videos(PG_DB) do |videos|
     popular_videos = videos
+    sleep 1.minutes
+    Fiber.yield
   end
 end
 
@@ -3644,9 +3648,23 @@ get "/videoplayback" do |env|
   host = "https://r#{fvip}---#{mn}.googlevideo.com"
   url = "/videoplayback?#{query_params.to_s}"
 
+  headers = env.request.headers
+  headers.delete("Host")
+  headers.delete("Cookie")
+  headers.delete("User-Agent")
+  headers.delete("Referer")
+
   region = query_params["region"]?
+
+  response = HTTP::Client::Response.new(403)
+  loop do
+    begin
       client = make_client(URI.parse(host), proxies, region)
-  response = client.head(url)
+      response = client.head(url, headers)
+      break
+    rescue ex
+    end
+  end
 
   if response.headers["Location"]?
     url = URI.parse(response.headers["Location"])
@@ -3664,12 +3682,6 @@ get "/videoplayback" do |env|
     halt env, status_code: 403
   end
 
-  headers = env.request.headers
-  headers.delete("Host")
-  headers.delete("Cookie")
-  headers.delete("User-Agent")
-  headers.delete("Referer")
-
   client = make_client(URI.parse(host), proxies, region)
   client.get(url, headers) do |response|
     env.response.status_code = response.status_code

src/invidious/jobs.cr

@@ -1,4 +1,4 @@
-def crawl_videos(db)
+def crawl_videos(db, logger)
   ids = Deque(String).new
   random = Random.new
 
@@ -21,7 +21,7 @@ def crawl_videos(db)
       id = ids[0]
       video = get_video(id, db)
     rescue ex
-      STDOUT << id << " : " << ex.message << "\n"
+      logger.write("#{id} : #{ex.message}\n")
       next
     ensure
       ids.delete(id)
@@ -46,7 +46,7 @@ def crawl_videos(db)
   end
 end
 
-def refresh_channels(db, max_threads = 1, full_refresh = false)
+def refresh_channels(db, logger, max_threads = 1, full_refresh = false)
   max_channel = Channel(Int32).new
 
   spawn do
@@ -73,7 +73,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false)
 
               db.exec("UPDATE channels SET updated = $1, author = $2 WHERE id = $3", Time.now, channel.author, id)
             rescue ex
-              STDOUT << id << " : " << ex.message << "\n"
+              logger.write("#{id} : #{ex.message}\n")
             end
 
             active_channel.send(true)
@@ -86,7 +86,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false)
   max_channel.send(max_threads)
 end
 
-def refresh_videos(db)
+def refresh_videos(db, logger)
   loop do
     db.query("SELECT id FROM videos ORDER BY updated") do |rs|
       rs.each do
@@ -94,7 +94,7 @@ def refresh_videos(db)
           id = rs.read(String)
           video = get_video(id, db)
         rescue ex
-          STDOUT << id << " : " << ex.message << "\n"
+          logger.write("#{id} : #{ex.message}\n")
           next
         end
       end
@@ -104,7 +104,7 @@ def refresh_videos(db)
   end
 end
 
-def refresh_feeds(db, max_threads = 1)
+def refresh_feeds(db, logger, max_threads = 1)
   max_channel = Channel(Int32).new
 
   spawn do
@@ -129,7 +129,7 @@ def refresh_feeds(db, max_threads = 1)
             begin
               db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
             rescue ex
-              STDOUT << "REFRESH " << email << " : " << ex.message << "\n"
+              logger.write("REFRESH #{email} : #{ex.message}\n")
             end
 
             active_channel.send(true)