From d0f421388815d630e4a13786c793d7e2fbe7749c Mon Sep 17 00:00:00 2001 From: Vladislav Yarmak Date: Thu, 14 Nov 2019 19:54:26 +0200 Subject: [PATCH] autogen: finished linux README generator --- drivers.json | 279 +++++++++--------- tools/readme_autogen/.gitignore | 107 +++++++ tools/readme_autogen/readme_autogen.py | 75 +++++ .../templates/linux_link_row.tmpl | 1 + .../templates/linux_nolink_row.tmpl | 1 + .../templates/linux_readme_master.tmpl | 110 +++++++ 6 files changed, 436 insertions(+), 137 deletions(-) create mode 100644 tools/readme_autogen/.gitignore create mode 100755 tools/readme_autogen/readme_autogen.py create mode 100644 tools/readme_autogen/templates/linux_link_row.tmpl create mode 100644 tools/readme_autogen/templates/linux_nolink_row.tmpl create mode 100644 tools/readme_autogen/templates/linux_readme_master.tmpl diff --git a/drivers.json b/drivers.json index 34bd84e..212a66a 100644 --- a/drivers.json +++ b/drivers.json @@ -1,142 +1,147 @@ { "linux": { - "x86_64": [ - { - "version": "375.39", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/375.39/NVIDIA-Linux-x86_64-375.39.run" - }, - { - "version": "390.77", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/390.77/NVIDIA-Linux-x86_64-390.77.run" - }, - { - "version": "390.87", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/390.87/NVIDIA-Linux-x86_64-390.87.run" - }, - { - "version": "396.24", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/396.24/NVIDIA-Linux-x86_64-396.24.run" - }, - { - "version": "396.26", - "driver_url": "https://international.download.nvidia.com/tesla/396.26/NVIDIA-Linux-x86_64-396.26.run" - }, - { - "version": "396.37", - "driver_url": "https://international.download.nvidia.com/tesla/396.37/NVIDIA-Linux-x86_64-396.37.run" - }, - { - "version": "396.54", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/396.54/NVIDIA-Linux-x86_64-396.54.run" - }, - { - "version": "410.48" - }, - { - "version": "410.57", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/410.57/NVIDIA-Linux-x86_64-410.57.run" - }, - { - "version": "410.73", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/410.73/NVIDIA-Linux-x86_64-410.73.run" - }, - { - "version": "410.78", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/410.78/NVIDIA-Linux-x86_64-410.78.run" - }, - { - "version": "410.79", - "driver_url": "https://international.download.nvidia.com/tesla/410.79/NVIDIA-Linux-x86_64-410.79.run" - }, - { - "version": "410.93", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/410.93/NVIDIA-Linux-x86_64-410.93.run" - }, - { - "version": "410.104", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/410.104/NVIDIA-Linux-x86_64-410.104.run" - }, - { - "version": "415.18", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/415.18/NVIDIA-Linux-x86_64-415.18.run" - }, - { - "version": "415.25", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/415.25/NVIDIA-Linux-x86_64-415.25.run" - }, - { - "version": "415.27", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/415.27/NVIDIA-Linux-x86_64-415.27.run" - }, - { - "version": "418.30", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/418.30/NVIDIA-Linux-x86_64-418.30.run" - }, - { - "version": "418.43", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/418.43/NVIDIA-Linux-x86_64-418.43.run" - }, - { - "version": "418.56", - "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/418.56/NVIDIA-Linux-x86_64-418.56.run" - }, - { - "version": "418.67" - }, - { - "version": "418.74", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/418.74/NVIDIA-Linux-x86_64-418.74.run" - }, - { - "version": "418.87.00" - }, - { - "version": "418.87.01" - }, - { - "version": "418.88", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/418.88/NVIDIA-Linux-x86_64-418.88.run" - }, - { - "version": "430.09", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/430.09/NVIDIA-Linux-x86_64-430.09.run" - }, - { - "version": "430.14", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/430.14/NVIDIA-Linux-x86_64-430.14.run" - }, - { - "version": "430.26", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/430.26/NVIDIA-Linux-x86_64-430.26.run" - }, - { - "version": "430.34", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/430.34/NVIDIA-Linux-x86_64-430.34.run" - }, - { - "version": "430.40", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/430.40/NVIDIA-Linux-x86_64-430.40.run" - }, - { - "version": "430.50", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/430.50/NVIDIA-Linux-x86_64-430.50.run" - }, - { - "version": "435.17", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/435.17/NVIDIA-Linux-x86_64-435.17.run" - }, - { - "version": "435.21", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/435.21/NVIDIA-Linux-x86_64-435.21.run" - }, - { - "version": "440.26", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/440.26/NVIDIA-Linux-x86_64-440.26.run" - }, - { - "version": "440.31", - "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/440.31/NVIDIA-Linux-x86_64-440.31.run" + "x86_64": { + "drivers": [ + { + "version": "375.39", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/375.39/NVIDIA-Linux-x86_64-375.39.run" + }, + { + "version": "390.77", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/390.77/NVIDIA-Linux-x86_64-390.77.run" + }, + { + "version": "390.87", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/390.87/NVIDIA-Linux-x86_64-390.87.run" + }, + { + "version": "396.24", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/396.24/NVIDIA-Linux-x86_64-396.24.run" + }, + { + "version": "396.26", + "driver_url": "https://international.download.nvidia.com/tesla/396.26/NVIDIA-Linux-x86_64-396.26.run" + }, + { + "version": "396.37", + "driver_url": "https://international.download.nvidia.com/tesla/396.37/NVIDIA-Linux-x86_64-396.37.run" + }, + { + "version": "396.54", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/396.54/NVIDIA-Linux-x86_64-396.54.run" + }, + { + "version": "410.48" + }, + { + "version": "410.57", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/410.57/NVIDIA-Linux-x86_64-410.57.run" + }, + { + "version": "410.73", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/410.73/NVIDIA-Linux-x86_64-410.73.run" + }, + { + "version": "410.78", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/410.78/NVIDIA-Linux-x86_64-410.78.run" + }, + { + "version": "410.79", + "driver_url": "https://international.download.nvidia.com/tesla/410.79/NVIDIA-Linux-x86_64-410.79.run" + }, + { + "version": "410.93", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/410.93/NVIDIA-Linux-x86_64-410.93.run" + }, + { + "version": "410.104", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/410.104/NVIDIA-Linux-x86_64-410.104.run" + }, + { + "version": "415.18", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/415.18/NVIDIA-Linux-x86_64-415.18.run" + }, + { + "version": "415.25", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/415.25/NVIDIA-Linux-x86_64-415.25.run" + }, + { + "version": "415.27", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/415.27/NVIDIA-Linux-x86_64-415.27.run" + }, + { + "version": "418.30", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/418.30/NVIDIA-Linux-x86_64-418.30.run" + }, + { + "version": "418.43", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/418.43/NVIDIA-Linux-x86_64-418.43.run" + }, + { + "version": "418.56", + "driver_url": "https://download.nvidia.com/XFree86/Linux-x86_64/418.56/NVIDIA-Linux-x86_64-418.56.run" + }, + { + "version": "418.67" + }, + { + "version": "418.74", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/418.74/NVIDIA-Linux-x86_64-418.74.run" + }, + { + "version": "418.87.00" + }, + { + "version": "418.87.01" + }, + { + "version": "418.88", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/418.88/NVIDIA-Linux-x86_64-418.88.run" + }, + { + "version": "430.09", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/430.09/NVIDIA-Linux-x86_64-430.09.run" + }, + { + "version": "430.14", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/430.14/NVIDIA-Linux-x86_64-430.14.run" + }, + { + "version": "430.26", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/430.26/NVIDIA-Linux-x86_64-430.26.run" + }, + { + "version": "430.34", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/430.34/NVIDIA-Linux-x86_64-430.34.run" + }, + { + "version": "430.40", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/430.40/NVIDIA-Linux-x86_64-430.40.run" + }, + { + "version": "430.50", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/430.50/NVIDIA-Linux-x86_64-430.50.run" + }, + { + "version": "435.17", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/435.17/NVIDIA-Linux-x86_64-435.17.run" + }, + { + "version": "435.21", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/435.21/NVIDIA-Linux-x86_64-435.21.run" + }, + { + "version": "440.26", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/440.26/NVIDIA-Linux-x86_64-440.26.run" + }, + { + "version": "440.31", + "driver_url": "https://international.download.nvidia.com/XFree86/Linux-x86_64/440.31/NVIDIA-Linux-x86_64-440.31.run" + } + ], + "example": { + "version": "430.50" } - ] + } } } diff --git a/tools/readme_autogen/.gitignore b/tools/readme_autogen/.gitignore new file mode 100644 index 0000000..9cfdd43 --- /dev/null +++ b/tools/readme_autogen/.gitignore @@ -0,0 +1,107 @@ +# Do not stage vim swapfiles to commit +*.swp + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/tools/readme_autogen/readme_autogen.py b/tools/readme_autogen/readme_autogen.py new file mode 100755 index 0000000..e23df47 --- /dev/null +++ b/tools/readme_autogen/readme_autogen.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import json +import os.path +from string import Template +from pprint import pprint + +BASE_PATH = os.path.dirname(os.path.abspath(__file__)) +TEMPLATE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "templates") +DATAFILE_PATH = os.path.join(BASE_PATH, + "..", "..", "drivers.json") +LINUX_README_PATH = os.path.join(BASE_PATH, + "..", "..", "README.md") +ENCODING="utf-8" + +def template(filename): + filename = os.path.join(TEMPLATE_PATH, filename) + with open(filename, encoding=ENCODING) as f: + t = Template(f.read()) + return t + +def version_key_fun(ver): + return tuple(map(int, ver.split('.'))) + +def find_driver(drivers, version, low=0, hi=None): + """ Bisect search on sorted drivers list """ + if hi is None: + hi = len(drivers) + L = hi - low + if L == 0: + return None + elif L == 1: + return drivers[low] if drivers[low]['version'] == version else None + else: + vkf_left = version_key_fun(drivers[low + L // 2]['version']) + vkf_right = version_key_fun(version) + if vkf_left < vkf_right: + return find_driver(drivers, version, low + L // 2, hi) + elif vkf_left > vkf_right: + return find_driver(drivers, version, low, low + L // 2) + else: + return drivers[low + L // 2] + +def linux_readme(data): + master_tmpl = template("linux_readme_master.tmpl") + linux_nolink_row_tmpl = template('linux_nolink_row.tmpl') + linux_link_row_tmpl = template('linux_link_row.tmpl') + drivers = sorted(data['drivers'], + key=lambda d: version_key_fun(d['version'])) + def row_gen(): + for drv in drivers: + driver_url = drv.get('driver_url') + t = linux_nolink_row_tmpl if driver_url is None else linux_link_row_tmpl + yield t.substitute(driver_version=drv['version'], + driver_url=driver_url).rstrip('\r\n') + version_list = "\n".join(row_gen()) + latest_version = drivers[-1]['version'] + example_driver = find_driver(drivers, data['example']['version']) + example_driver_url = example_driver['driver_url'] + return master_tmpl.substitute(version_list=version_list, + latest_version=latest_version, + example_driver_url=example_driver_url, + example_driver_version=example_driver['version'], + example_driver_file=os.path.basename(example_driver_url)) + +def main(): + with open(DATAFILE_PATH) as data_file: + data = json.load(data_file) + res = linux_readme(data['linux']['x86_64']) + with open(LINUX_README_PATH, 'w', encoding=ENCODING) as out: + out.write(res) + +if __name__ == '__main__': + main() diff --git a/tools/readme_autogen/templates/linux_link_row.tmpl b/tools/readme_autogen/templates/linux_link_row.tmpl new file mode 100644 index 0000000..4613098 --- /dev/null +++ b/tools/readme_autogen/templates/linux_link_row.tmpl @@ -0,0 +1 @@ + - [$driver_version]($driver_url) diff --git a/tools/readme_autogen/templates/linux_nolink_row.tmpl b/tools/readme_autogen/templates/linux_nolink_row.tmpl new file mode 100644 index 0000000..741a30f --- /dev/null +++ b/tools/readme_autogen/templates/linux_nolink_row.tmpl @@ -0,0 +1 @@ + - $driver_version diff --git a/tools/readme_autogen/templates/linux_readme_master.tmpl b/tools/readme_autogen/templates/linux_readme_master.tmpl new file mode 100644 index 0000000..7be4101 --- /dev/null +++ b/tools/readme_autogen/templates/linux_readme_master.tmpl @@ -0,0 +1,110 @@ +# nvidia-patch + +![GitHub last commit](https://img.shields.io/github/last-commit/keylase/nvidia-patch.svg) ![Latest version](https://img.shields.io/badge/latest%20linux%20driver%20version-${latest_version}-brightgreen.svg) + +This patch removes restriction on maximum number of simultaneous NVENC video encoding sessions imposed by Nvidia to consumer-grade GPUs. + +Main target operating system is **GNU/Linux**, but for **Windows** support see [**win**](win). + +Requirements: +- x86\_64 system architecture +- GNU/Linux operating system +- nvenc-compatible gpu (https://developer.nvidia.com/video-encode-decode-gpu-support-matrix#Encoder) +- Nvidia driver. Patch available for: +$version_list + +## Synopsis + +``` +# bash ./patch.sh -h + +SYNOPSIS + patch.sh [-s] [-r|-h|-c VERSION|-l] + +DESCRIPTION + The patch for Nvidia drivers to remove NVENC session limit + + -s Silent mode (No output) + -r Rollback to original (Restore lib from backup) + -h Print this help message + -c VERSION Check if version VERSION supported by this patch. + Returns true exit code (0) if version is supported. + -l List supported driver versions + +``` + +## Step-by-Step guide + +Examples are provided for driver version ${example_driver_version}. All commands executed as root. + +### Download and install driver + +Skip this step if you already have installed driver with version supported by this patch (from distro packages, for example). + +Make sure you have kernel headers and compiler installed before running Nvidia driver installer. Kernel headers and compiler are required to build nvidia kernel module. Recommended way to do this is to install `dkms` package, if it is available in your distro. This way `dkms` package will pull all required dependencies to allow building kernel modules and kernel module builds will be automated in a reliable fashion. + +```bash +mkdir /opt/nvidia && cd /opt/nvidia +wget ${example_driver_url} +chmod +x ./${example_driver_file} +./${example_driver_file} +``` + +### Check driver + +```bash +nvidia-smi +``` + +Output should show no errors and details about your driver and GPU. + +### Patch driver + +This patch performs backup of original file prior to making changes. + +```bash +bash ./patch.sh +``` + +You're all set! + +## Rollback + +If something got broken you may restore patched driver from backup: + +```bash +bash ./patch.sh -r +``` + +## Docker support + +It is possible to use this patch with nvidia-docker containers, even if host machine hasn't patched drivers. See `Dockerfile` for example. + +Essentially all you need to do during build is: + +* `COPY` the `patch.sh` and `docker-entrypoint.sh` files into your container. +* Make sure `docker-entrypoint.sh` is invoked on container start. + +`docker-entrypoint.sh` script does on-the-fly patching by means of manipulating dynamic linker to workaround read-only mount of Nvidia runtime. Finally it passes original docker command to shell, like if entrypoint was not restricted by `ENTRYPOINT` directive. So `docker run --runtime=nvidia -it mycontainer echo 123` will print `123`. Also it can be just invoked from your entrypoint script, if you have any. + +## Benchmarks + +* [Plex Media Server: nVidia Hardware Transcoding Calculator for Plex Estimates](https://www.elpamsoft.com/?p=Plex-Hardware-Transcoding) - useful benchmark of achieved simultaneous transcodes with various stream quality and hardware with patched drivers. + +## See also + +* Plex Media Server: enable HW **decoding**: + * [GH Issue](https://github.com/keylase/nvidia-patch/issues/51) + * PMS Forum: + 1. [https://forums.plex.tv/t/hardware-accelerated-decode-nvidia-for-linux/233510/158](https://forums.plex.tv/t/hardware-accelerated-decode-nvidia-for-linux/233510/158) + 2. [https://forums.plex.tv/t/hardware-accelerated-decode-nvidia-for-linux/233510/172](https://forums.plex.tv/t/hardware-accelerated-decode-nvidia-for-linux/233510/172) +* Unraid / Docker: + * [GH Issue](https://github.com/keylase/nvidia-patch/issues/43) + * Unraid Nvidia plugin: + * [Repo](https://github.com/linuxserver/Unraid-Nvidia-Plugin) + * [Forum page](https://forums.unraid.net/topic/77813-plugin-linuxserverio-unraid-nvidia/) ([archive link](https://web.archive.org/web/20190211145338/https://forums.unraid.net/topic/77813-plugin-linuxserverio-unraid-nvidia/)) +* Original research behind this patch + * [Original post in Russian](https://habr.com/post/262563/) + * [3rd party English translation](https://weekly-geekly.github.io/articles/262563/index.html) + +If you experience `CreateBitstreamBuffer failed: out of memory (10)`, then you have to lower buffers number used for every encoding session. If you are using `ffmpeg`, see option `-surfaces` ("Number of concurrent surfaces") and try value near `-surfaces 8`.