+full refactor
+feat: configuration, progress bar, OSV
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
# created by virtualenv automatically
|
||||
*/ПЗ Сундук ЕА.docx
|
||||
*/Сундук ЕА_CVExplorer.pptx
|
||||
@@ -0,0 +1,87 @@
|
||||
# This file must be used with "source bin/activate" *from bash*
|
||||
# you cannot run it directly
|
||||
|
||||
|
||||
if [ "${BASH_SOURCE-}" = "$0" ]; then
|
||||
echo "You must source this script: \$ source $0" >&2
|
||||
exit 33
|
||||
fi
|
||||
|
||||
deactivate () {
|
||||
unset -f pydoc >/dev/null 2>&1 || true
|
||||
|
||||
# reset old environment variables
|
||||
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
|
||||
if ! [ -z "${_OLD_VIRTUAL_PATH:+_}" ] ; then
|
||||
PATH="$_OLD_VIRTUAL_PATH"
|
||||
export PATH
|
||||
unset _OLD_VIRTUAL_PATH
|
||||
fi
|
||||
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
|
||||
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
|
||||
export PYTHONHOME
|
||||
unset _OLD_VIRTUAL_PYTHONHOME
|
||||
fi
|
||||
|
||||
# The hash command must be called to get it to forget past
|
||||
# commands. Without forgetting past commands the $PATH changes
|
||||
# we made may not be respected
|
||||
hash -r 2>/dev/null
|
||||
|
||||
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
|
||||
PS1="$_OLD_VIRTUAL_PS1"
|
||||
export PS1
|
||||
unset _OLD_VIRTUAL_PS1
|
||||
fi
|
||||
|
||||
unset VIRTUAL_ENV
|
||||
unset VIRTUAL_ENV_PROMPT
|
||||
if [ ! "${1-}" = "nondestructive" ] ; then
|
||||
# Self destruct!
|
||||
unset -f deactivate
|
||||
fi
|
||||
}
|
||||
|
||||
# unset irrelevant variables
|
||||
deactivate nondestructive
|
||||
|
||||
VIRTUAL_ENV='/home/chest/PycharmProjects/CVExplorer/.venv'
|
||||
if ([ "$OSTYPE" = "cygwin" ] || [ "$OSTYPE" = "msys" ]) && $(command -v cygpath &> /dev/null) ; then
|
||||
VIRTUAL_ENV=$(cygpath -u "$VIRTUAL_ENV")
|
||||
fi
|
||||
export VIRTUAL_ENV
|
||||
|
||||
_OLD_VIRTUAL_PATH="$PATH"
|
||||
PATH="$VIRTUAL_ENV/bin:$PATH"
|
||||
export PATH
|
||||
|
||||
if [ "x" != x ] ; then
|
||||
VIRTUAL_ENV_PROMPT=""
|
||||
else
|
||||
VIRTUAL_ENV_PROMPT=$(basename "$VIRTUAL_ENV")
|
||||
fi
|
||||
export VIRTUAL_ENV_PROMPT
|
||||
|
||||
# unset PYTHONHOME if set
|
||||
if ! [ -z "${PYTHONHOME+_}" ] ; then
|
||||
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
|
||||
unset PYTHONHOME
|
||||
fi
|
||||
|
||||
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
|
||||
_OLD_VIRTUAL_PS1="${PS1-}"
|
||||
PS1="(${VIRTUAL_ENV_PROMPT}) ${PS1-}"
|
||||
export PS1
|
||||
fi
|
||||
|
||||
# Make sure to unalias pydoc if it's already there
|
||||
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
|
||||
|
||||
pydoc () {
|
||||
python -m pydoc "$@"
|
||||
}
|
||||
|
||||
# The hash command must be called to get it to forget past
|
||||
# commands. Without forgetting past commands the $PATH changes
|
||||
# we made may not be respected
|
||||
hash -r 2>/dev/null
|
||||
@@ -0,0 +1,55 @@
|
||||
# This file must be used with "source bin/activate.csh" *from csh*.
|
||||
# You cannot run it directly.
|
||||
# Created by Davide Di Blasi <davidedb@gmail.com>.
|
||||
|
||||
set newline='\
|
||||
'
|
||||
|
||||
alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH:q" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT:q" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc'
|
||||
|
||||
# Unset irrelevant variables.
|
||||
deactivate nondestructive
|
||||
|
||||
setenv VIRTUAL_ENV '/home/chest/PycharmProjects/CVExplorer/.venv'
|
||||
|
||||
set _OLD_VIRTUAL_PATH="$PATH:q"
|
||||
setenv PATH "$VIRTUAL_ENV:q/bin:$PATH:q"
|
||||
|
||||
|
||||
|
||||
if ('' != "") then
|
||||
setenv VIRTUAL_ENV_PROMPT ''
|
||||
else
|
||||
setenv VIRTUAL_ENV_PROMPT "$VIRTUAL_ENV:t:q"
|
||||
endif
|
||||
|
||||
if ( $?VIRTUAL_ENV_DISABLE_PROMPT ) then
|
||||
if ( $VIRTUAL_ENV_DISABLE_PROMPT == "" ) then
|
||||
set do_prompt = "1"
|
||||
else
|
||||
set do_prompt = "0"
|
||||
endif
|
||||
else
|
||||
set do_prompt = "1"
|
||||
endif
|
||||
|
||||
if ( $do_prompt == "1" ) then
|
||||
# Could be in a non-interactive environment,
|
||||
# in which case, $prompt is undefined and we wouldn't
|
||||
# care about the prompt anyway.
|
||||
if ( $?prompt ) then
|
||||
set _OLD_VIRTUAL_PROMPT="$prompt:q"
|
||||
if ( "$prompt:q" =~ *"$newline:q"* ) then
|
||||
:
|
||||
else
|
||||
set prompt = '('"$VIRTUAL_ENV_PROMPT:q"') '"$prompt:q"
|
||||
endif
|
||||
endif
|
||||
endif
|
||||
|
||||
unset env_name
|
||||
unset do_prompt
|
||||
|
||||
alias pydoc python -m pydoc
|
||||
|
||||
rehash
|
||||
@@ -0,0 +1,103 @@
|
||||
# This file must be used using `source bin/activate.fish` *within a running fish ( http://fishshell.com ) session*.
|
||||
# Do not run it directly.
|
||||
|
||||
function _bashify_path -d "Converts a fish path to something bash can recognize"
|
||||
set fishy_path $argv
|
||||
set bashy_path $fishy_path[1]
|
||||
for path_part in $fishy_path[2..-1]
|
||||
set bashy_path "$bashy_path:$path_part"
|
||||
end
|
||||
echo $bashy_path
|
||||
end
|
||||
|
||||
function _fishify_path -d "Converts a bash path to something fish can recognize"
|
||||
echo $argv | tr ':' '\n'
|
||||
end
|
||||
|
||||
function deactivate -d 'Exit virtualenv mode and return to the normal environment.'
|
||||
# reset old environment variables
|
||||
if test -n "$_OLD_VIRTUAL_PATH"
|
||||
# https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling
|
||||
if test (echo $FISH_VERSION | head -c 1) -lt 3
|
||||
set -gx PATH (_fishify_path "$_OLD_VIRTUAL_PATH")
|
||||
else
|
||||
set -gx PATH $_OLD_VIRTUAL_PATH
|
||||
end
|
||||
set -e _OLD_VIRTUAL_PATH
|
||||
end
|
||||
|
||||
if test -n "$_OLD_VIRTUAL_PYTHONHOME"
|
||||
set -gx PYTHONHOME "$_OLD_VIRTUAL_PYTHONHOME"
|
||||
set -e _OLD_VIRTUAL_PYTHONHOME
|
||||
end
|
||||
|
||||
if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
|
||||
and functions -q _old_fish_prompt
|
||||
# Set an empty local `$fish_function_path` to allow the removal of `fish_prompt` using `functions -e`.
|
||||
set -l fish_function_path
|
||||
|
||||
# Erase virtualenv's `fish_prompt` and restore the original.
|
||||
functions -e fish_prompt
|
||||
functions -c _old_fish_prompt fish_prompt
|
||||
functions -e _old_fish_prompt
|
||||
set -e _OLD_FISH_PROMPT_OVERRIDE
|
||||
end
|
||||
|
||||
set -e VIRTUAL_ENV
|
||||
set -e VIRTUAL_ENV_PROMPT
|
||||
|
||||
if test "$argv[1]" != 'nondestructive'
|
||||
# Self-destruct!
|
||||
functions -e pydoc
|
||||
functions -e deactivate
|
||||
functions -e _bashify_path
|
||||
functions -e _fishify_path
|
||||
end
|
||||
end
|
||||
|
||||
# Unset irrelevant variables.
|
||||
deactivate nondestructive
|
||||
|
||||
set -gx VIRTUAL_ENV '/home/chest/PycharmProjects/CVExplorer/.venv'
|
||||
|
||||
# https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling
|
||||
if test (echo $FISH_VERSION | head -c 1) -lt 3
|
||||
set -gx _OLD_VIRTUAL_PATH (_bashify_path $PATH)
|
||||
else
|
||||
set -gx _OLD_VIRTUAL_PATH $PATH
|
||||
end
|
||||
set -gx PATH "$VIRTUAL_ENV"'/bin' $PATH
|
||||
|
||||
# Prompt override provided?
|
||||
# If not, just use the environment name.
|
||||
if test -n ''
|
||||
set -gx VIRTUAL_ENV_PROMPT ''
|
||||
else
|
||||
set -gx VIRTUAL_ENV_PROMPT (basename "$VIRTUAL_ENV")
|
||||
end
|
||||
|
||||
# Unset `$PYTHONHOME` if set.
|
||||
if set -q PYTHONHOME
|
||||
set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
|
||||
set -e PYTHONHOME
|
||||
end
|
||||
|
||||
function pydoc
|
||||
python -m pydoc $argv
|
||||
end
|
||||
|
||||
if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
|
||||
# Copy the current `fish_prompt` function as `_old_fish_prompt`.
|
||||
functions -c fish_prompt _old_fish_prompt
|
||||
|
||||
function fish_prompt
|
||||
# Run the user's prompt first; it might depend on (pipe)status.
|
||||
set -l prompt (_old_fish_prompt)
|
||||
|
||||
printf '(%s) ' $VIRTUAL_ENV_PROMPT
|
||||
|
||||
string join -- \n $prompt # handle multi-line prompts
|
||||
end
|
||||
|
||||
set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
|
||||
end
|
||||
@@ -0,0 +1,96 @@
|
||||
# virtualenv activation module
|
||||
# Activate with `overlay use activate.nu`
|
||||
# Deactivate with `deactivate`, as usual
|
||||
#
|
||||
# To customize the overlay name, you can call `overlay use activate.nu as foo`,
|
||||
# but then simply `deactivate` won't work because it is just an alias to hide
|
||||
# the "activate" overlay. You'd need to call `overlay hide foo` manually.
|
||||
|
||||
export-env {
|
||||
def is-string [x] {
|
||||
($x | describe) == 'string'
|
||||
}
|
||||
|
||||
def has-env [...names] {
|
||||
$names | each {|n|
|
||||
$n in $env
|
||||
} | all {|i| $i == true}
|
||||
}
|
||||
|
||||
# Emulates a `test -z`, but btter as it handles e.g 'false'
|
||||
def is-env-true [name: string] {
|
||||
if (has-env $name) {
|
||||
# Try to parse 'true', '0', '1', and fail if not convertible
|
||||
let parsed = (do -i { $env | get $name | into bool })
|
||||
if ($parsed | describe) == 'bool' {
|
||||
$parsed
|
||||
} else {
|
||||
not ($env | get -i $name | is-empty)
|
||||
}
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
let virtual_env = '/home/chest/PycharmProjects/CVExplorer/.venv'
|
||||
let bin = 'bin'
|
||||
|
||||
let is_windows = ($nu.os-info.family) == 'windows'
|
||||
let path_name = (if (has-env 'Path') {
|
||||
'Path'
|
||||
} else {
|
||||
'PATH'
|
||||
}
|
||||
)
|
||||
|
||||
let venv_path = ([$virtual_env $bin] | path join)
|
||||
let new_path = ($env | get $path_name | prepend $venv_path)
|
||||
|
||||
# If there is no default prompt, then use the env name instead
|
||||
let virtual_env_prompt = (if ('' | is-empty) {
|
||||
($virtual_env | path basename)
|
||||
} else {
|
||||
''
|
||||
})
|
||||
|
||||
let new_env = {
|
||||
$path_name : $new_path
|
||||
VIRTUAL_ENV : $virtual_env
|
||||
VIRTUAL_ENV_PROMPT : $virtual_env_prompt
|
||||
}
|
||||
|
||||
let new_env = (if (is-env-true 'VIRTUAL_ENV_DISABLE_PROMPT') {
|
||||
$new_env
|
||||
} else {
|
||||
# Creating the new prompt for the session
|
||||
let virtual_prefix = $'(char lparen)($virtual_env_prompt)(char rparen) '
|
||||
|
||||
# Back up the old prompt builder
|
||||
let old_prompt_command = (if (has-env 'PROMPT_COMMAND') {
|
||||
$env.PROMPT_COMMAND
|
||||
} else {
|
||||
''
|
||||
})
|
||||
|
||||
let new_prompt = (if (has-env 'PROMPT_COMMAND') {
|
||||
if 'closure' in ($old_prompt_command | describe) {
|
||||
{|| $'($virtual_prefix)(do $old_prompt_command)' }
|
||||
} else {
|
||||
{|| $'($virtual_prefix)($old_prompt_command)' }
|
||||
}
|
||||
} else {
|
||||
{|| $'($virtual_prefix)' }
|
||||
})
|
||||
|
||||
$new_env | merge {
|
||||
PROMPT_COMMAND : $new_prompt
|
||||
VIRTUAL_PREFIX : $virtual_prefix
|
||||
}
|
||||
})
|
||||
|
||||
# Environment variables that will be loaded as the virtual env
|
||||
load-env $new_env
|
||||
}
|
||||
|
||||
export alias pydoc = python -m pydoc
|
||||
export alias deactivate = overlay hide activate
|
||||
@@ -0,0 +1,61 @@
|
||||
$script:THIS_PATH = $myinvocation.mycommand.path
|
||||
$script:BASE_DIR = Split-Path (Resolve-Path "$THIS_PATH/..") -Parent
|
||||
|
||||
function global:deactivate([switch] $NonDestructive) {
|
||||
if (Test-Path variable:_OLD_VIRTUAL_PATH) {
|
||||
$env:PATH = $variable:_OLD_VIRTUAL_PATH
|
||||
Remove-Variable "_OLD_VIRTUAL_PATH" -Scope global
|
||||
}
|
||||
|
||||
if (Test-Path function:_old_virtual_prompt) {
|
||||
$function:prompt = $function:_old_virtual_prompt
|
||||
Remove-Item function:\_old_virtual_prompt
|
||||
}
|
||||
|
||||
if ($env:VIRTUAL_ENV) {
|
||||
Remove-Item env:VIRTUAL_ENV -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
if ($env:VIRTUAL_ENV_PROMPT) {
|
||||
Remove-Item env:VIRTUAL_ENV_PROMPT -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
if (!$NonDestructive) {
|
||||
# Self destruct!
|
||||
Remove-Item function:deactivate
|
||||
Remove-Item function:pydoc
|
||||
}
|
||||
}
|
||||
|
||||
function global:pydoc {
|
||||
python -m pydoc $args
|
||||
}
|
||||
|
||||
# unset irrelevant variables
|
||||
deactivate -nondestructive
|
||||
|
||||
$VIRTUAL_ENV = $BASE_DIR
|
||||
$env:VIRTUAL_ENV = $VIRTUAL_ENV
|
||||
|
||||
if ("" -ne "") {
|
||||
$env:VIRTUAL_ENV_PROMPT = ""
|
||||
}
|
||||
else {
|
||||
$env:VIRTUAL_ENV_PROMPT = $( Split-Path $env:VIRTUAL_ENV -Leaf )
|
||||
}
|
||||
|
||||
New-Variable -Scope global -Name _OLD_VIRTUAL_PATH -Value $env:PATH
|
||||
|
||||
$env:PATH = "$env:VIRTUAL_ENV/bin:" + $env:PATH
|
||||
if (!$env:VIRTUAL_ENV_DISABLE_PROMPT) {
|
||||
function global:_old_virtual_prompt {
|
||||
""
|
||||
}
|
||||
$function:_old_virtual_prompt = $function:prompt
|
||||
|
||||
function global:prompt {
|
||||
# Add the custom prefix to the existing prompt
|
||||
$previous_prompt_value = & $function:_old_virtual_prompt
|
||||
("(" + $env:VIRTUAL_ENV_PROMPT + ") " + $previous_prompt_value)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
"""
|
||||
Activate virtualenv for current interpreter:
|
||||
|
||||
Use exec(open(this_file).read(), {'__file__': this_file}).
|
||||
|
||||
This can be used when you must use an existing Python interpreter, not the virtualenv bin/python.
|
||||
""" # noqa: D415
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import site
|
||||
import sys
|
||||
|
||||
try:
|
||||
abs_file = os.path.abspath(__file__)
|
||||
except NameError as exc:
|
||||
msg = "You must use exec(open(this_file).read(), {'__file__': this_file}))"
|
||||
raise AssertionError(msg) from exc
|
||||
|
||||
bin_dir = os.path.dirname(abs_file)
|
||||
base = bin_dir[: -len("bin") - 1] # strip away the bin part from the __file__, plus the path separator
|
||||
|
||||
# prepend bin to PATH (this file is inside the bin directory)
|
||||
os.environ["PATH"] = os.pathsep.join([bin_dir, *os.environ.get("PATH", "").split(os.pathsep)])
|
||||
os.environ["VIRTUAL_ENV"] = base # virtual env is right above bin directory
|
||||
os.environ["VIRTUAL_ENV_PROMPT"] = "" or os.path.basename(base) # noqa: SIM222
|
||||
|
||||
# add the virtual environments libraries to the host python import mechanism
|
||||
prev_length = len(sys.path)
|
||||
for lib in "../lib/python3.13/site-packages".split(os.pathsep):
|
||||
path = os.path.realpath(os.path.join(bin_dir, lib))
|
||||
site.addsitedir(path.decode("utf-8") if "" else path)
|
||||
sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
|
||||
|
||||
sys.real_prefix = sys.prefix
|
||||
sys.prefix = base
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/home/chest/PycharmProjects/CVExplorer/.venv/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
import re
|
||||
import sys
|
||||
from charset_normalizer.cli import cli_detect
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(cli_detect())
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/home/chest/PycharmProjects/CVExplorer/.venv/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
import re
|
||||
import sys
|
||||
from pip._internal.cli.main import main
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(main())
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/home/chest/PycharmProjects/CVExplorer/.venv/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
import re
|
||||
import sys
|
||||
from pip._internal.cli.main import main
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(main())
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/home/chest/PycharmProjects/CVExplorer/.venv/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
import re
|
||||
import sys
|
||||
from pip._internal.cli.main import main
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(main())
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/home/chest/PycharmProjects/CVExplorer/.venv/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
import re
|
||||
import sys
|
||||
from pip._internal.cli.main import main
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(main())
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
/etc/profiles/per-user/chest/bin/python3
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
python
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
python
|
||||
@@ -0,0 +1 @@
|
||||
import _virtualenv
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Patches that are applied at runtime to the virtual environment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from contextlib import suppress
|
||||
|
||||
VIRTUALENV_PATCH_FILE = os.path.join(__file__)
|
||||
|
||||
|
||||
def patch_dist(dist):
|
||||
"""
|
||||
Distutils allows user to configure some arguments via a configuration file:
|
||||
https://docs.python.org/3/install/index.html#distutils-configuration-files.
|
||||
|
||||
Some of this arguments though don't make sense in context of the virtual environment files, let's fix them up.
|
||||
""" # noqa: D205
|
||||
# we cannot allow some install config as that would get packages installed outside of the virtual environment
|
||||
old_parse_config_files = dist.Distribution.parse_config_files
|
||||
|
||||
def parse_config_files(self, *args, **kwargs):
|
||||
result = old_parse_config_files(self, *args, **kwargs)
|
||||
install = self.get_option_dict("install")
|
||||
|
||||
if "prefix" in install: # the prefix governs where to install the libraries
|
||||
install["prefix"] = VIRTUALENV_PATCH_FILE, os.path.abspath(sys.prefix)
|
||||
for base in ("purelib", "platlib", "headers", "scripts", "data"):
|
||||
key = f"install_{base}"
|
||||
if key in install: # do not allow global configs to hijack venv paths
|
||||
install.pop(key, None)
|
||||
return result
|
||||
|
||||
dist.Distribution.parse_config_files = parse_config_files
|
||||
|
||||
|
||||
# Import hook that patches some modules to ignore configuration values that break package installation in case
|
||||
# of virtual environments.
|
||||
_DISTUTILS_PATCH = "distutils.dist", "setuptools.dist"
|
||||
# https://docs.python.org/3/library/importlib.html#setting-up-an-importer
|
||||
|
||||
|
||||
class _Finder:
|
||||
"""A meta path finder that allows patching the imported distutils modules."""
|
||||
|
||||
fullname = None
|
||||
|
||||
# lock[0] is threading.Lock(), but initialized lazily to avoid importing threading very early at startup,
|
||||
# because there are gevent-based applications that need to be first to import threading by themselves.
|
||||
# See https://github.com/pypa/virtualenv/issues/1895 for details.
|
||||
lock = [] # noqa: RUF012
|
||||
|
||||
def find_spec(self, fullname, path, target=None): # noqa: ARG002
|
||||
if fullname in _DISTUTILS_PATCH and self.fullname is None:
|
||||
# initialize lock[0] lazily
|
||||
if len(self.lock) == 0:
|
||||
import threading
|
||||
|
||||
lock = threading.Lock()
|
||||
# there is possibility that two threads T1 and T2 are simultaneously running into find_spec,
|
||||
# observing .lock as empty, and further going into hereby initialization. However due to the GIL,
|
||||
# list.append() operation is atomic and this way only one of the threads will "win" to put the lock
|
||||
# - that every thread will use - into .lock[0].
|
||||
# https://docs.python.org/3/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe
|
||||
self.lock.append(lock)
|
||||
|
||||
from functools import partial
|
||||
from importlib.util import find_spec
|
||||
|
||||
with self.lock[0]:
|
||||
self.fullname = fullname
|
||||
try:
|
||||
spec = find_spec(fullname, path)
|
||||
if spec is not None:
|
||||
# https://www.python.org/dev/peps/pep-0451/#how-loading-will-work
|
||||
is_new_api = hasattr(spec.loader, "exec_module")
|
||||
func_name = "exec_module" if is_new_api else "load_module"
|
||||
old = getattr(spec.loader, func_name)
|
||||
func = self.exec_module if is_new_api else self.load_module
|
||||
if old is not func:
|
||||
with suppress(AttributeError): # C-Extension loaders are r/o such as zipimporter with <3.7
|
||||
setattr(spec.loader, func_name, partial(func, old))
|
||||
return spec
|
||||
finally:
|
||||
self.fullname = None
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def exec_module(old, module):
|
||||
old(module)
|
||||
if module.__name__ in _DISTUTILS_PATCH:
|
||||
patch_dist(module)
|
||||
|
||||
@staticmethod
|
||||
def load_module(old, name):
|
||||
module = old(name)
|
||||
if module.__name__ in _DISTUTILS_PATCH:
|
||||
patch_dist(module)
|
||||
return module
|
||||
|
||||
|
||||
sys.meta_path.insert(0, _Finder())
|
||||
@@ -0,0 +1 @@
|
||||
pip
|
||||
@@ -0,0 +1,78 @@
|
||||
Metadata-Version: 2.4
|
||||
Name: certifi
|
||||
Version: 2025.11.12
|
||||
Summary: Python package for providing Mozilla's CA Bundle.
|
||||
Home-page: https://github.com/certifi/python-certifi
|
||||
Author: Kenneth Reitz
|
||||
Author-email: me@kennethreitz.com
|
||||
License: MPL-2.0
|
||||
Project-URL: Source, https://github.com/certifi/python-certifi
|
||||
Classifier: Development Status :: 5 - Production/Stable
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
|
||||
Classifier: Natural Language :: English
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3 :: Only
|
||||
Classifier: Programming Language :: Python :: 3.7
|
||||
Classifier: Programming Language :: Python :: 3.8
|
||||
Classifier: Programming Language :: Python :: 3.9
|
||||
Classifier: Programming Language :: Python :: 3.10
|
||||
Classifier: Programming Language :: Python :: 3.11
|
||||
Classifier: Programming Language :: Python :: 3.12
|
||||
Classifier: Programming Language :: Python :: 3.13
|
||||
Classifier: Programming Language :: Python :: 3.14
|
||||
Requires-Python: >=3.7
|
||||
License-File: LICENSE
|
||||
Dynamic: author
|
||||
Dynamic: author-email
|
||||
Dynamic: classifier
|
||||
Dynamic: description
|
||||
Dynamic: home-page
|
||||
Dynamic: license
|
||||
Dynamic: license-file
|
||||
Dynamic: project-url
|
||||
Dynamic: requires-python
|
||||
Dynamic: summary
|
||||
|
||||
Certifi: Python SSL Certificates
|
||||
================================
|
||||
|
||||
Certifi provides Mozilla's carefully curated collection of Root Certificates for
|
||||
validating the trustworthiness of SSL certificates while verifying the identity
|
||||
of TLS hosts. It has been extracted from the `Requests`_ project.
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
``certifi`` is available on PyPI. Simply install it with ``pip``::
|
||||
|
||||
$ pip install certifi
|
||||
|
||||
Usage
|
||||
-----
|
||||
|
||||
To reference the installed certificate authority (CA) bundle, you can use the
|
||||
built-in function::
|
||||
|
||||
>>> import certifi
|
||||
|
||||
>>> certifi.where()
|
||||
'/usr/local/lib/python3.7/site-packages/certifi/cacert.pem'
|
||||
|
||||
Or from the command line::
|
||||
|
||||
$ python -m certifi
|
||||
/usr/local/lib/python3.7/site-packages/certifi/cacert.pem
|
||||
|
||||
Enjoy!
|
||||
|
||||
.. _`Requests`: https://requests.readthedocs.io/en/master/
|
||||
|
||||
Addition/Removal of Certificates
|
||||
--------------------------------
|
||||
|
||||
Certifi does not support any addition/removal or other modification of the
|
||||
CA trust store content. This project is intended to provide a reliable and
|
||||
highly portable root of trust to python deployments. Look to upstream projects
|
||||
for methods to use alternate trust.
|
||||
@@ -0,0 +1,14 @@
|
||||
certifi-2025.11.12.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
|
||||
certifi-2025.11.12.dist-info/METADATA,sha256=_JprGu_1lWSdHlruRBKcorXnrfvBDhvX_6KRr8HQbLc,2475
|
||||
certifi-2025.11.12.dist-info/RECORD,,
|
||||
certifi-2025.11.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
||||
certifi-2025.11.12.dist-info/licenses/LICENSE,sha256=6TcW2mucDVpKHfYP5pWzcPBpVgPSH2-D8FPkLPwQyvc,989
|
||||
certifi-2025.11.12.dist-info/top_level.txt,sha256=KMu4vUCfsjLrkPbSNdgdekS-pVJzBAJFO__nI8NF6-U,8
|
||||
certifi/__init__.py,sha256=1BRSxNMnZW7CZ2oJtYWLoJgfHfcB9i273exwiPwfjJM,94
|
||||
certifi/__main__.py,sha256=xBBoj905TUWBLRGANOcf7oi6e-3dMP4cEoG9OyMs11g,243
|
||||
certifi/__pycache__/__init__.cpython-313.pyc,,
|
||||
certifi/__pycache__/__main__.cpython-313.pyc,,
|
||||
certifi/__pycache__/core.cpython-313.pyc,,
|
||||
certifi/cacert.pem,sha256=oa1dZD4hxDtb7XTH4IkdzbWPavUcis4eTwINZUqlKhY,283932
|
||||
certifi/core.py,sha256=XFXycndG5pf37ayeF8N32HUuDafsyhkVMbO4BAPWHa0,3394
|
||||
certifi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||
@@ -0,0 +1,5 @@
|
||||
Wheel-Version: 1.0
|
||||
Generator: setuptools (80.9.0)
|
||||
Root-Is-Purelib: true
|
||||
Tag: py3-none-any
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
This package contains a modified version of ca-bundle.crt:
|
||||
|
||||
ca-bundle.crt -- Bundle of CA Root Certificates
|
||||
|
||||
This is a bundle of X.509 certificates of public Certificate Authorities
|
||||
(CA). These were automatically extracted from Mozilla's root certificates
|
||||
file (certdata.txt). This file can be found in the mozilla source tree:
|
||||
https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/ckfw/builtins/certdata.txt
|
||||
It contains the certificates in PEM format and therefore
|
||||
can be directly used with curl / libcurl / php_curl, or with
|
||||
an Apache+mod_ssl webserver for SSL client authentication.
|
||||
Just configure this file as the SSLCACertificateFile.#
|
||||
|
||||
***** BEGIN LICENSE BLOCK *****
|
||||
This Source Code Form is subject to the terms of the Mozilla Public License,
|
||||
v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain
|
||||
one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
***** END LICENSE BLOCK *****
|
||||
@(#) $RCSfile: certdata.txt,v $ $Revision: 1.80 $ $Date: 2011/11/03 15:11:58 $
|
||||
@@ -0,0 +1 @@
|
||||
certifi
|
||||
@@ -0,0 +1,4 @@
|
||||
from .core import contents, where
|
||||
|
||||
__all__ = ["contents", "where"]
|
||||
__version__ = "2025.11.12"
|
||||
@@ -0,0 +1,12 @@
|
||||
import argparse
|
||||
|
||||
from certifi import contents, where
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-c", "--contents", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.contents:
|
||||
print(contents())
|
||||
else:
|
||||
print(where())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,83 @@
|
||||
"""
|
||||
certifi.py
|
||||
~~~~~~~~~~
|
||||
|
||||
This module returns the installation location of cacert.pem or its contents.
|
||||
"""
|
||||
import sys
|
||||
import atexit
|
||||
|
||||
def exit_cacert_ctx() -> None:
|
||||
_CACERT_CTX.__exit__(None, None, None) # type: ignore[union-attr]
|
||||
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
|
||||
from importlib.resources import as_file, files
|
||||
|
||||
_CACERT_CTX = None
|
||||
_CACERT_PATH = None
|
||||
|
||||
def where() -> str:
|
||||
# This is slightly terrible, but we want to delay extracting the file
|
||||
# in cases where we're inside of a zipimport situation until someone
|
||||
# actually calls where(), but we don't want to re-extract the file
|
||||
# on every call of where(), so we'll do it once then store it in a
|
||||
# global variable.
|
||||
global _CACERT_CTX
|
||||
global _CACERT_PATH
|
||||
if _CACERT_PATH is None:
|
||||
# This is slightly janky, the importlib.resources API wants you to
|
||||
# manage the cleanup of this file, so it doesn't actually return a
|
||||
# path, it returns a context manager that will give you the path
|
||||
# when you enter it and will do any cleanup when you leave it. In
|
||||
# the common case of not needing a temporary file, it will just
|
||||
# return the file system location and the __exit__() is a no-op.
|
||||
#
|
||||
# We also have to hold onto the actual context manager, because
|
||||
# it will do the cleanup whenever it gets garbage collected, so
|
||||
# we will also store that at the global level as well.
|
||||
_CACERT_CTX = as_file(files("certifi").joinpath("cacert.pem"))
|
||||
_CACERT_PATH = str(_CACERT_CTX.__enter__())
|
||||
atexit.register(exit_cacert_ctx)
|
||||
|
||||
return _CACERT_PATH
|
||||
|
||||
def contents() -> str:
|
||||
return files("certifi").joinpath("cacert.pem").read_text(encoding="ascii")
|
||||
|
||||
else:
|
||||
|
||||
from importlib.resources import path as get_path, read_text
|
||||
|
||||
_CACERT_CTX = None
|
||||
_CACERT_PATH = None
|
||||
|
||||
def where() -> str:
|
||||
# This is slightly terrible, but we want to delay extracting the
|
||||
# file in cases where we're inside of a zipimport situation until
|
||||
# someone actually calls where(), but we don't want to re-extract
|
||||
# the file on every call of where(), so we'll do it once then store
|
||||
# it in a global variable.
|
||||
global _CACERT_CTX
|
||||
global _CACERT_PATH
|
||||
if _CACERT_PATH is None:
|
||||
# This is slightly janky, the importlib.resources API wants you
|
||||
# to manage the cleanup of this file, so it doesn't actually
|
||||
# return a path, it returns a context manager that will give
|
||||
# you the path when you enter it and will do any cleanup when
|
||||
# you leave it. In the common case of not needing a temporary
|
||||
# file, it will just return the file system location and the
|
||||
# __exit__() is a no-op.
|
||||
#
|
||||
# We also have to hold onto the actual context manager, because
|
||||
# it will do the cleanup whenever it gets garbage collected, so
|
||||
# we will also store that at the global level as well.
|
||||
_CACERT_CTX = get_path("certifi", "cacert.pem")
|
||||
_CACERT_PATH = str(_CACERT_CTX.__enter__())
|
||||
atexit.register(exit_cacert_ctx)
|
||||
|
||||
return _CACERT_PATH
|
||||
|
||||
def contents() -> str:
|
||||
return read_text("certifi", "cacert.pem", encoding="ascii")
|
||||
@@ -0,0 +1 @@
|
||||
pip
|
||||
@@ -0,0 +1,764 @@
|
||||
Metadata-Version: 2.4
|
||||
Name: charset-normalizer
|
||||
Version: 3.4.4
|
||||
Summary: The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet.
|
||||
Author-email: "Ahmed R. TAHRI" <tahri.ahmed@proton.me>
|
||||
Maintainer-email: "Ahmed R. TAHRI" <tahri.ahmed@proton.me>
|
||||
License: MIT
|
||||
Project-URL: Changelog, https://github.com/jawah/charset_normalizer/blob/master/CHANGELOG.md
|
||||
Project-URL: Documentation, https://charset-normalizer.readthedocs.io/
|
||||
Project-URL: Code, https://github.com/jawah/charset_normalizer
|
||||
Project-URL: Issue tracker, https://github.com/jawah/charset_normalizer/issues
|
||||
Keywords: encoding,charset,charset-detector,detector,normalization,unicode,chardet,detect
|
||||
Classifier: Development Status :: 5 - Production/Stable
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: Operating System :: OS Independent
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3.7
|
||||
Classifier: Programming Language :: Python :: 3.8
|
||||
Classifier: Programming Language :: Python :: 3.9
|
||||
Classifier: Programming Language :: Python :: 3.10
|
||||
Classifier: Programming Language :: Python :: 3.11
|
||||
Classifier: Programming Language :: Python :: 3.12
|
||||
Classifier: Programming Language :: Python :: 3.13
|
||||
Classifier: Programming Language :: Python :: 3.14
|
||||
Classifier: Programming Language :: Python :: 3 :: Only
|
||||
Classifier: Programming Language :: Python :: Implementation :: CPython
|
||||
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
||||
Classifier: Topic :: Text Processing :: Linguistic
|
||||
Classifier: Topic :: Utilities
|
||||
Classifier: Typing :: Typed
|
||||
Requires-Python: >=3.7
|
||||
Description-Content-Type: text/markdown
|
||||
License-File: LICENSE
|
||||
Provides-Extra: unicode-backport
|
||||
Dynamic: license-file
|
||||
|
||||
<h1 align="center">Charset Detection, for Everyone 👋</h1>
|
||||
|
||||
<p align="center">
|
||||
<sup>The Real First Universal Charset Detector</sup><br>
|
||||
<a href="https://pypi.org/project/charset-normalizer">
|
||||
<img src="https://img.shields.io/pypi/pyversions/charset_normalizer.svg?orange=blue" />
|
||||
</a>
|
||||
<a href="https://pepy.tech/project/charset-normalizer/">
|
||||
<img alt="Download Count Total" src="https://static.pepy.tech/badge/charset-normalizer/month" />
|
||||
</a>
|
||||
<a href="https://bestpractices.coreinfrastructure.org/projects/7297">
|
||||
<img src="https://bestpractices.coreinfrastructure.org/projects/7297/badge">
|
||||
</a>
|
||||
</p>
|
||||
<p align="center">
|
||||
<sup><i>Featured Packages</i></sup><br>
|
||||
<a href="https://github.com/jawah/niquests">
|
||||
<img alt="Static Badge" src="https://img.shields.io/badge/Niquests-Most_Advanced_HTTP_Client-cyan">
|
||||
</a>
|
||||
<a href="https://github.com/jawah/wassima">
|
||||
<img alt="Static Badge" src="https://img.shields.io/badge/Wassima-Certifi_Replacement-cyan">
|
||||
</a>
|
||||
</p>
|
||||
<p align="center">
|
||||
<sup><i>In other language (unofficial port - by the community)</i></sup><br>
|
||||
<a href="https://github.com/nickspring/charset-normalizer-rs">
|
||||
<img alt="Static Badge" src="https://img.shields.io/badge/Rust-red">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
> A library that helps you read text from an unknown charset encoding.<br /> Motivated by `chardet`,
|
||||
> I'm trying to resolve the issue by taking a new approach.
|
||||
> All IANA character set names for which the Python core library provides codecs are supported.
|
||||
|
||||
<p align="center">
|
||||
>>>>> <a href="https://charsetnormalizerweb.ousret.now.sh" target="_blank">👉 Try Me Online Now, Then Adopt Me 👈 </a> <<<<<
|
||||
</p>
|
||||
|
||||
This project offers you an alternative to **Universal Charset Encoding Detector**, also known as **Chardet**.
|
||||
|
||||
| Feature | [Chardet](https://github.com/chardet/chardet) | Charset Normalizer | [cChardet](https://github.com/PyYoshi/cChardet) |
|
||||
|--------------------------------------------------|:---------------------------------------------:|:--------------------------------------------------------------------------------------------------:|:-----------------------------------------------:|
|
||||
| `Fast` | ❌ | ✅ | ✅ |
|
||||
| `Universal**` | ❌ | ✅ | ❌ |
|
||||
| `Reliable` **without** distinguishable standards | ❌ | ✅ | ✅ |
|
||||
| `Reliable` **with** distinguishable standards | ✅ | ✅ | ✅ |
|
||||
| `License` | LGPL-2.1<br>_restrictive_ | MIT | MPL-1.1<br>_restrictive_ |
|
||||
| `Native Python` | ✅ | ✅ | ❌ |
|
||||
| `Detect spoken language` | ❌ | ✅ | N/A |
|
||||
| `UnicodeDecodeError Safety` | ❌ | ✅ | ❌ |
|
||||
| `Whl Size (min)` | 193.6 kB | 42 kB | ~200 kB |
|
||||
| `Supported Encoding` | 33 | 🎉 [99](https://charset-normalizer.readthedocs.io/en/latest/user/support.html#supported-encodings) | 40 |
|
||||
|
||||
<p align="center">
|
||||
<img src="https://i.imgflip.com/373iay.gif" alt="Reading Normalized Text" width="226"/><img src="https://media.tenor.com/images/c0180f70732a18b4965448d33adba3d0/tenor.gif" alt="Cat Reading Text" width="200"/>
|
||||
</p>
|
||||
|
||||
*\*\* : They are clearly using specific code for a specific encoding even if covering most of used one*<br>
|
||||
|
||||
## ⚡ Performance
|
||||
|
||||
This package offer better performance than its counterpart Chardet. Here are some numbers.
|
||||
|
||||
| Package | Accuracy | Mean per file (ms) | File per sec (est) |
|
||||
|-----------------------------------------------|:--------:|:------------------:|:------------------:|
|
||||
| [chardet](https://github.com/chardet/chardet) | 86 % | 63 ms | 16 file/sec |
|
||||
| charset-normalizer | **98 %** | **10 ms** | 100 file/sec |
|
||||
|
||||
| Package | 99th percentile | 95th percentile | 50th percentile |
|
||||
|-----------------------------------------------|:---------------:|:---------------:|:---------------:|
|
||||
| [chardet](https://github.com/chardet/chardet) | 265 ms | 71 ms | 7 ms |
|
||||
| charset-normalizer | 100 ms | 50 ms | 5 ms |
|
||||
|
||||
_updated as of december 2024 using CPython 3.12_
|
||||
|
||||
Chardet's performance on larger file (1MB+) are very poor. Expect huge difference on large payload.
|
||||
|
||||
> Stats are generated using 400+ files using default parameters. More details on used files, see GHA workflows.
|
||||
> And yes, these results might change at any time. The dataset can be updated to include more files.
|
||||
> The actual delays heavily depends on your CPU capabilities. The factors should remain the same.
|
||||
> Keep in mind that the stats are generous and that Chardet accuracy vs our is measured using Chardet initial capability
|
||||
> (e.g. Supported Encoding) Challenge-them if you want.
|
||||
|
||||
## ✨ Installation
|
||||
|
||||
Using pip:
|
||||
|
||||
```sh
|
||||
pip install charset-normalizer -U
|
||||
```
|
||||
|
||||
## 🚀 Basic Usage
|
||||
|
||||
### CLI
|
||||
This package comes with a CLI.
|
||||
|
||||
```
|
||||
usage: normalizer [-h] [-v] [-a] [-n] [-m] [-r] [-f] [-t THRESHOLD]
|
||||
file [file ...]
|
||||
|
||||
The Real First Universal Charset Detector. Discover originating encoding used
|
||||
on text file. Normalize text to unicode.
|
||||
|
||||
positional arguments:
|
||||
files File(s) to be analysed
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-v, --verbose Display complementary information about file if any.
|
||||
Stdout will contain logs about the detection process.
|
||||
-a, --with-alternative
|
||||
Output complementary possibilities if any. Top-level
|
||||
JSON WILL be a list.
|
||||
-n, --normalize Permit to normalize input file. If not set, program
|
||||
does not write anything.
|
||||
-m, --minimal Only output the charset detected to STDOUT. Disabling
|
||||
JSON output.
|
||||
-r, --replace Replace file when trying to normalize it instead of
|
||||
creating a new one.
|
||||
-f, --force Replace file without asking if you are sure, use this
|
||||
flag with caution.
|
||||
-t THRESHOLD, --threshold THRESHOLD
|
||||
Define a custom maximum amount of chaos allowed in
|
||||
decoded content. 0. <= chaos <= 1.
|
||||
--version Show version information and exit.
|
||||
```
|
||||
|
||||
```bash
|
||||
normalizer ./data/sample.1.fr.srt
|
||||
```
|
||||
|
||||
or
|
||||
|
||||
```bash
|
||||
python -m charset_normalizer ./data/sample.1.fr.srt
|
||||
```
|
||||
|
||||
🎉 Since version 1.4.0 the CLI produce easily usable stdout result in JSON format.
|
||||
|
||||
```json
|
||||
{
|
||||
"path": "/home/default/projects/charset_normalizer/data/sample.1.fr.srt",
|
||||
"encoding": "cp1252",
|
||||
"encoding_aliases": [
|
||||
"1252",
|
||||
"windows_1252"
|
||||
],
|
||||
"alternative_encodings": [
|
||||
"cp1254",
|
||||
"cp1256",
|
||||
"cp1258",
|
||||
"iso8859_14",
|
||||
"iso8859_15",
|
||||
"iso8859_16",
|
||||
"iso8859_3",
|
||||
"iso8859_9",
|
||||
"latin_1",
|
||||
"mbcs"
|
||||
],
|
||||
"language": "French",
|
||||
"alphabets": [
|
||||
"Basic Latin",
|
||||
"Latin-1 Supplement"
|
||||
],
|
||||
"has_sig_or_bom": false,
|
||||
"chaos": 0.149,
|
||||
"coherence": 97.152,
|
||||
"unicode_path": null,
|
||||
"is_preferred": true
|
||||
}
|
||||
```
|
||||
|
||||
### Python
|
||||
*Just print out normalized text*
|
||||
```python
|
||||
from charset_normalizer import from_path
|
||||
|
||||
results = from_path('./my_subtitle.srt')
|
||||
|
||||
print(str(results.best()))
|
||||
```
|
||||
|
||||
*Upgrade your code without effort*
|
||||
```python
|
||||
from charset_normalizer import detect
|
||||
```
|
||||
|
||||
The above code will behave the same as **chardet**. We ensure that we offer the best (reasonable) BC result possible.
|
||||
|
||||
See the docs for advanced usage : [readthedocs.io](https://charset-normalizer.readthedocs.io/en/latest/)
|
||||
|
||||
## 😇 Why
|
||||
|
||||
When I started using Chardet, I noticed that it was not suited to my expectations, and I wanted to propose a
|
||||
reliable alternative using a completely different method. Also! I never back down on a good challenge!
|
||||
|
||||
I **don't care** about the **originating charset** encoding, because **two different tables** can
|
||||
produce **two identical rendered string.**
|
||||
What I want is to get readable text, the best I can.
|
||||
|
||||
In a way, **I'm brute forcing text decoding.** How cool is that ? 😎
|
||||
|
||||
Don't confuse package **ftfy** with charset-normalizer or chardet. ftfy goal is to repair Unicode string whereas charset-normalizer to convert raw file in unknown encoding to unicode.
|
||||
|
||||
## 🍰 How
|
||||
|
||||
- Discard all charset encoding table that could not fit the binary content.
|
||||
- Measure noise, or the mess once opened (by chunks) with a corresponding charset encoding.
|
||||
- Extract matches with the lowest mess detected.
|
||||
- Additionally, we measure coherence / probe for a language.
|
||||
|
||||
**Wait a minute**, what is noise/mess and coherence according to **YOU ?**
|
||||
|
||||
*Noise :* I opened hundred of text files, **written by humans**, with the wrong encoding table. **I observed**, then
|
||||
**I established** some ground rules about **what is obvious** when **it seems like** a mess (aka. defining noise in rendered text).
|
||||
I know that my interpretation of what is noise is probably incomplete, feel free to contribute in order to
|
||||
improve or rewrite it.
|
||||
|
||||
*Coherence :* For each language there is on earth, we have computed ranked letter appearance occurrences (the best we can). So I thought
|
||||
that intel is worth something here. So I use those records against decoded text to check if I can detect intelligent design.
|
||||
|
||||
## ⚡ Known limitations
|
||||
|
||||
- Language detection is unreliable when text contains two or more languages sharing identical letters. (eg. HTML (english tags) + Turkish content (Sharing Latin characters))
|
||||
- Every charset detector heavily depends on sufficient content. In common cases, do not bother run detection on very tiny content.
|
||||
|
||||
## ⚠️ About Python EOLs
|
||||
|
||||
**If you are running:**
|
||||
|
||||
- Python >=2.7,<3.5: Unsupported
|
||||
- Python 3.5: charset-normalizer < 2.1
|
||||
- Python 3.6: charset-normalizer < 3.1
|
||||
- Python 3.7: charset-normalizer < 4.0
|
||||
|
||||
Upgrade your Python interpreter as soon as possible.
|
||||
|
||||
## 👤 Contributing
|
||||
|
||||
Contributions, issues and feature requests are very much welcome.<br />
|
||||
Feel free to check [issues page](https://github.com/ousret/charset_normalizer/issues) if you want to contribute.
|
||||
|
||||
## 📝 License
|
||||
|
||||
Copyright © [Ahmed TAHRI @Ousret](https://github.com/Ousret).<br />
|
||||
This project is [MIT](https://github.com/Ousret/charset_normalizer/blob/master/LICENSE) licensed.
|
||||
|
||||
Characters frequencies used in this project © 2012 [Denny Vrandečić](http://simia.net/letters/)
|
||||
|
||||
## 💼 For Enterprise
|
||||
|
||||
Professional support for charset-normalizer is available as part of the [Tidelift
|
||||
Subscription][1]. Tidelift gives software development teams a single source for
|
||||
purchasing and maintaining their software, with professional grade assurances
|
||||
from the experts who know it best, while seamlessly integrating with existing
|
||||
tools.
|
||||
|
||||
[1]: https://tidelift.com/subscription/pkg/pypi-charset-normalizer?utm_source=pypi-charset-normalizer&utm_medium=readme
|
||||
|
||||
[](https://www.bestpractices.dev/projects/7297)
|
||||
|
||||
# Changelog
|
||||
All notable changes to charset-normalizer will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
||||
|
||||
## [3.4.4](https://github.com/Ousret/charset_normalizer/compare/3.4.2...3.4.4) (2025-10-13)
|
||||
|
||||
### Changed
|
||||
- Bound `setuptools` to a specific constraint `setuptools>=68,<=81`.
|
||||
- Raised upper bound of mypyc for the optional pre-built extension to v1.18.2
|
||||
|
||||
### Removed
|
||||
- `setuptools-scm` as a build dependency.
|
||||
|
||||
### Misc
|
||||
- Enforced hashes in `dev-requirements.txt` and created `ci-requirements.txt` for security purposes.
|
||||
- Additional pre-built wheels for riscv64, s390x, and armv7l architectures.
|
||||
- Restore ` multiple.intoto.jsonl` in GitHub releases in addition to individual attestation file per wheel.
|
||||
|
||||
## [3.4.3](https://github.com/Ousret/charset_normalizer/compare/3.4.2...3.4.3) (2025-08-09)
|
||||
|
||||
### Changed
|
||||
- mypy(c) is no longer a required dependency at build time if `CHARSET_NORMALIZER_USE_MYPYC` isn't set to `1`. (#595) (#583)
|
||||
- automatically lower confidence on small bytes samples that are not Unicode in `detect` output legacy function. (#391)
|
||||
|
||||
### Added
|
||||
- Custom build backend to overcome inability to mark mypy as an optional dependency in the build phase.
|
||||
- Support for Python 3.14
|
||||
|
||||
### Fixed
|
||||
- sdist archive contained useless directories.
|
||||
- automatically fallback on valid UTF-16 or UTF-32 even if the md says it's noisy. (#633)
|
||||
|
||||
### Misc
|
||||
- SBOM are automatically published to the relevant GitHub release to comply with regulatory changes.
|
||||
Each published wheel comes with its SBOM. We choose CycloneDX as the format.
|
||||
- Prebuilt optimized wheel are no longer distributed by default for CPython 3.7 due to a change in cibuildwheel.
|
||||
|
||||
## [3.4.2](https://github.com/Ousret/charset_normalizer/compare/3.4.1...3.4.2) (2025-05-02)
|
||||
|
||||
### Fixed
|
||||
- Addressed the DeprecationWarning in our CLI regarding `argparse.FileType` by backporting the target class into the package. (#591)
|
||||
- Improved the overall reliability of the detector with CJK Ideographs. (#605) (#587)
|
||||
|
||||
### Changed
|
||||
- Optional mypyc compilation upgraded to version 1.15 for Python >= 3.8
|
||||
|
||||
## [3.4.1](https://github.com/Ousret/charset_normalizer/compare/3.4.0...3.4.1) (2024-12-24)
|
||||
|
||||
### Changed
|
||||
- Project metadata are now stored using `pyproject.toml` instead of `setup.cfg` using setuptools as the build backend.
|
||||
- Enforce annotation delayed loading for a simpler and consistent types in the project.
|
||||
- Optional mypyc compilation upgraded to version 1.14 for Python >= 3.8
|
||||
|
||||
### Added
|
||||
- pre-commit configuration.
|
||||
- noxfile.
|
||||
|
||||
### Removed
|
||||
- `build-requirements.txt` as per using `pyproject.toml` native build configuration.
|
||||
- `bin/integration.py` and `bin/serve.py` in favor of downstream integration test (see noxfile).
|
||||
- `setup.cfg` in favor of `pyproject.toml` metadata configuration.
|
||||
- Unused `utils.range_scan` function.
|
||||
|
||||
### Fixed
|
||||
- Converting content to Unicode bytes may insert `utf_8` instead of preferred `utf-8`. (#572)
|
||||
- Deprecation warning "'count' is passed as positional argument" when converting to Unicode bytes on Python 3.13+
|
||||
|
||||
## [3.4.0](https://github.com/Ousret/charset_normalizer/compare/3.3.2...3.4.0) (2024-10-08)
|
||||
|
||||
### Added
|
||||
- Argument `--no-preemptive` in the CLI to prevent the detector to search for hints.
|
||||
- Support for Python 3.13 (#512)
|
||||
|
||||
### Fixed
|
||||
- Relax the TypeError exception thrown when trying to compare a CharsetMatch with anything else than a CharsetMatch.
|
||||
- Improved the general reliability of the detector based on user feedbacks. (#520) (#509) (#498) (#407) (#537)
|
||||
- Declared charset in content (preemptive detection) not changed when converting to utf-8 bytes. (#381)
|
||||
|
||||
## [3.3.2](https://github.com/Ousret/charset_normalizer/compare/3.3.1...3.3.2) (2023-10-31)
|
||||
|
||||
### Fixed
|
||||
- Unintentional memory usage regression when using large payload that match several encoding (#376)
|
||||
- Regression on some detection case showcased in the documentation (#371)
|
||||
|
||||
### Added
|
||||
- Noise (md) probe that identify malformed arabic representation due to the presence of letters in isolated form (credit to my wife)
|
||||
|
||||
## [3.3.1](https://github.com/Ousret/charset_normalizer/compare/3.3.0...3.3.1) (2023-10-22)
|
||||
|
||||
### Changed
|
||||
- Optional mypyc compilation upgraded to version 1.6.1 for Python >= 3.8
|
||||
- Improved the general detection reliability based on reports from the community
|
||||
|
||||
## [3.3.0](https://github.com/Ousret/charset_normalizer/compare/3.2.0...3.3.0) (2023-09-30)
|
||||
|
||||
### Added
|
||||
- Allow to execute the CLI (e.g. normalizer) through `python -m charset_normalizer.cli` or `python -m charset_normalizer`
|
||||
- Support for 9 forgotten encoding that are supported by Python but unlisted in `encoding.aliases` as they have no alias (#323)
|
||||
|
||||
### Removed
|
||||
- (internal) Redundant utils.is_ascii function and unused function is_private_use_only
|
||||
- (internal) charset_normalizer.assets is moved inside charset_normalizer.constant
|
||||
|
||||
### Changed
|
||||
- (internal) Unicode code blocks in constants are updated using the latest v15.0.0 definition to improve detection
|
||||
- Optional mypyc compilation upgraded to version 1.5.1 for Python >= 3.8
|
||||
|
||||
### Fixed
|
||||
- Unable to properly sort CharsetMatch when both chaos/noise and coherence were close due to an unreachable condition in \_\_lt\_\_ (#350)
|
||||
|
||||
## [3.2.0](https://github.com/Ousret/charset_normalizer/compare/3.1.0...3.2.0) (2023-06-07)
|
||||
|
||||
### Changed
|
||||
- Typehint for function `from_path` no longer enforce `PathLike` as its first argument
|
||||
- Minor improvement over the global detection reliability
|
||||
|
||||
### Added
|
||||
- Introduce function `is_binary` that relies on main capabilities, and optimized to detect binaries
|
||||
- Propagate `enable_fallback` argument throughout `from_bytes`, `from_path`, and `from_fp` that allow a deeper control over the detection (default True)
|
||||
- Explicit support for Python 3.12
|
||||
|
||||
### Fixed
|
||||
- Edge case detection failure where a file would contain 'very-long' camel cased word (Issue #289)
|
||||
|
||||
## [3.1.0](https://github.com/Ousret/charset_normalizer/compare/3.0.1...3.1.0) (2023-03-06)
|
||||
|
||||
### Added
|
||||
- Argument `should_rename_legacy` for legacy function `detect` and disregard any new arguments without errors (PR #262)
|
||||
|
||||
### Removed
|
||||
- Support for Python 3.6 (PR #260)
|
||||
|
||||
### Changed
|
||||
- Optional speedup provided by mypy/c 1.0.1
|
||||
|
||||
## [3.0.1](https://github.com/Ousret/charset_normalizer/compare/3.0.0...3.0.1) (2022-11-18)
|
||||
|
||||
### Fixed
|
||||
- Multi-bytes cutter/chunk generator did not always cut correctly (PR #233)
|
||||
|
||||
### Changed
|
||||
- Speedup provided by mypy/c 0.990 on Python >= 3.7
|
||||
|
||||
## [3.0.0](https://github.com/Ousret/charset_normalizer/compare/2.1.1...3.0.0) (2022-10-20)
|
||||
|
||||
### Added
|
||||
- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results
|
||||
- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES
|
||||
- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio
|
||||
- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl)
|
||||
|
||||
### Changed
|
||||
- Build with static metadata using 'build' frontend
|
||||
- Make the language detection stricter
|
||||
- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1
|
||||
|
||||
### Fixed
|
||||
- CLI with opt --normalize fail when using full path for files
|
||||
- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it
|
||||
- Sphinx warnings when generating the documentation
|
||||
|
||||
### Removed
|
||||
- Coherence detector no longer return 'Simple English' instead return 'English'
|
||||
- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese'
|
||||
- Breaking: Method `first()` and `best()` from CharsetMatch
|
||||
- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII)
|
||||
- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches
|
||||
- Breaking: Top-level function `normalize`
|
||||
- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch
|
||||
- Support for the backport `unicodedata2`
|
||||
|
||||
## [3.0.0rc1](https://github.com/Ousret/charset_normalizer/compare/3.0.0b2...3.0.0rc1) (2022-10-18)
|
||||
|
||||
### Added
|
||||
- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results
|
||||
- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES
|
||||
- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio
|
||||
|
||||
### Changed
|
||||
- Build with static metadata using 'build' frontend
|
||||
- Make the language detection stricter
|
||||
|
||||
### Fixed
|
||||
- CLI with opt --normalize fail when using full path for files
|
||||
- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it
|
||||
|
||||
### Removed
|
||||
- Coherence detector no longer return 'Simple English' instead return 'English'
|
||||
- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese'
|
||||
|
||||
## [3.0.0b2](https://github.com/Ousret/charset_normalizer/compare/3.0.0b1...3.0.0b2) (2022-08-21)
|
||||
|
||||
### Added
|
||||
- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl)
|
||||
|
||||
### Removed
|
||||
- Breaking: Method `first()` and `best()` from CharsetMatch
|
||||
- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII)
|
||||
|
||||
### Fixed
|
||||
- Sphinx warnings when generating the documentation
|
||||
|
||||
## [3.0.0b1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...3.0.0b1) (2022-08-15)
|
||||
|
||||
### Changed
|
||||
- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1
|
||||
|
||||
### Removed
|
||||
- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches
|
||||
- Breaking: Top-level function `normalize`
|
||||
- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch
|
||||
- Support for the backport `unicodedata2`
|
||||
|
||||
## [2.1.1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...2.1.1) (2022-08-19)
|
||||
|
||||
### Deprecated
|
||||
- Function `normalize` scheduled for removal in 3.0
|
||||
|
||||
### Changed
|
||||
- Removed useless call to decode in fn is_unprintable (#206)
|
||||
|
||||
### Fixed
|
||||
- Third-party library (i18n xgettext) crashing not recognizing utf_8 (PEP 263) with underscore from [@aleksandernovikov](https://github.com/aleksandernovikov) (#204)
|
||||
|
||||
## [2.1.0](https://github.com/Ousret/charset_normalizer/compare/2.0.12...2.1.0) (2022-06-19)
|
||||
|
||||
### Added
|
||||
- Output the Unicode table version when running the CLI with `--version` (PR #194)
|
||||
|
||||
### Changed
|
||||
- Re-use decoded buffer for single byte character sets from [@nijel](https://github.com/nijel) (PR #175)
|
||||
- Fixing some performance bottlenecks from [@deedy5](https://github.com/deedy5) (PR #183)
|
||||
|
||||
### Fixed
|
||||
- Workaround potential bug in cpython with Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space (PR #175)
|
||||
- CLI default threshold aligned with the API threshold from [@oleksandr-kuzmenko](https://github.com/oleksandr-kuzmenko) (PR #181)
|
||||
|
||||
### Removed
|
||||
- Support for Python 3.5 (PR #192)
|
||||
|
||||
### Deprecated
|
||||
- Use of backport unicodedata from `unicodedata2` as Python is quickly catching up, scheduled for removal in 3.0 (PR #194)
|
||||
|
||||
## [2.0.12](https://github.com/Ousret/charset_normalizer/compare/2.0.11...2.0.12) (2022-02-12)
|
||||
|
||||
### Fixed
|
||||
- ASCII miss-detection on rare cases (PR #170)
|
||||
|
||||
## [2.0.11](https://github.com/Ousret/charset_normalizer/compare/2.0.10...2.0.11) (2022-01-30)
|
||||
|
||||
### Added
|
||||
- Explicit support for Python 3.11 (PR #164)
|
||||
|
||||
### Changed
|
||||
- The logging behavior have been completely reviewed, now using only TRACE and DEBUG levels (PR #163 #165)
|
||||
|
||||
## [2.0.10](https://github.com/Ousret/charset_normalizer/compare/2.0.9...2.0.10) (2022-01-04)
|
||||
|
||||
### Fixed
|
||||
- Fallback match entries might lead to UnicodeDecodeError for large bytes sequence (PR #154)
|
||||
|
||||
### Changed
|
||||
- Skipping the language-detection (CD) on ASCII (PR #155)
|
||||
|
||||
## [2.0.9](https://github.com/Ousret/charset_normalizer/compare/2.0.8...2.0.9) (2021-12-03)
|
||||
|
||||
### Changed
|
||||
- Moderating the logging impact (since 2.0.8) for specific environments (PR #147)
|
||||
|
||||
### Fixed
|
||||
- Wrong logging level applied when setting kwarg `explain` to True (PR #146)
|
||||
|
||||
## [2.0.8](https://github.com/Ousret/charset_normalizer/compare/2.0.7...2.0.8) (2021-11-24)
|
||||
### Changed
|
||||
- Improvement over Vietnamese detection (PR #126)
|
||||
- MD improvement on trailing data and long foreign (non-pure latin) data (PR #124)
|
||||
- Efficiency improvements in cd/alphabet_languages from [@adbar](https://github.com/adbar) (PR #122)
|
||||
- call sum() without an intermediary list following PEP 289 recommendations from [@adbar](https://github.com/adbar) (PR #129)
|
||||
- Code style as refactored by Sourcery-AI (PR #131)
|
||||
- Minor adjustment on the MD around european words (PR #133)
|
||||
- Remove and replace SRTs from assets / tests (PR #139)
|
||||
- Initialize the library logger with a `NullHandler` by default from [@nmaynes](https://github.com/nmaynes) (PR #135)
|
||||
- Setting kwarg `explain` to True will add provisionally (bounded to function lifespan) a specific stream handler (PR #135)
|
||||
|
||||
### Fixed
|
||||
- Fix large (misleading) sequence giving UnicodeDecodeError (PR #137)
|
||||
- Avoid using too insignificant chunk (PR #137)
|
||||
|
||||
### Added
|
||||
- Add and expose function `set_logging_handler` to configure a specific StreamHandler from [@nmaynes](https://github.com/nmaynes) (PR #135)
|
||||
- Add `CHANGELOG.md` entries, format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) (PR #141)
|
||||
|
||||
## [2.0.7](https://github.com/Ousret/charset_normalizer/compare/2.0.6...2.0.7) (2021-10-11)
|
||||
### Added
|
||||
- Add support for Kazakh (Cyrillic) language detection (PR #109)
|
||||
|
||||
### Changed
|
||||
- Further, improve inferring the language from a given single-byte code page (PR #112)
|
||||
- Vainly trying to leverage PEP263 when PEP3120 is not supported (PR #116)
|
||||
- Refactoring for potential performance improvements in loops from [@adbar](https://github.com/adbar) (PR #113)
|
||||
- Various detection improvement (MD+CD) (PR #117)
|
||||
|
||||
### Removed
|
||||
- Remove redundant logging entry about detected language(s) (PR #115)
|
||||
|
||||
### Fixed
|
||||
- Fix a minor inconsistency between Python 3.5 and other versions regarding language detection (PR #117 #102)
|
||||
|
||||
## [2.0.6](https://github.com/Ousret/charset_normalizer/compare/2.0.5...2.0.6) (2021-09-18)
|
||||
### Fixed
|
||||
- Unforeseen regression with the loss of the backward-compatibility with some older minor of Python 3.5.x (PR #100)
|
||||
- Fix CLI crash when using --minimal output in certain cases (PR #103)
|
||||
|
||||
### Changed
|
||||
- Minor improvement to the detection efficiency (less than 1%) (PR #106 #101)
|
||||
|
||||
## [2.0.5](https://github.com/Ousret/charset_normalizer/compare/2.0.4...2.0.5) (2021-09-14)
|
||||
### Changed
|
||||
- The project now comply with: flake8, mypy, isort and black to ensure a better overall quality (PR #81)
|
||||
- The BC-support with v1.x was improved, the old staticmethods are restored (PR #82)
|
||||
- The Unicode detection is slightly improved (PR #93)
|
||||
- Add syntax sugar \_\_bool\_\_ for results CharsetMatches list-container (PR #91)
|
||||
|
||||
### Removed
|
||||
- The project no longer raise warning on tiny content given for detection, will be simply logged as warning instead (PR #92)
|
||||
|
||||
### Fixed
|
||||
- In some rare case, the chunks extractor could cut in the middle of a multi-byte character and could mislead the mess detection (PR #95)
|
||||
- Some rare 'space' characters could trip up the UnprintablePlugin/Mess detection (PR #96)
|
||||
- The MANIFEST.in was not exhaustive (PR #78)
|
||||
|
||||
## [2.0.4](https://github.com/Ousret/charset_normalizer/compare/2.0.3...2.0.4) (2021-07-30)
|
||||
### Fixed
|
||||
- The CLI no longer raise an unexpected exception when no encoding has been found (PR #70)
|
||||
- Fix accessing the 'alphabets' property when the payload contains surrogate characters (PR #68)
|
||||
- The logger could mislead (explain=True) on detected languages and the impact of one MBCS match (PR #72)
|
||||
- Submatch factoring could be wrong in rare edge cases (PR #72)
|
||||
- Multiple files given to the CLI were ignored when publishing results to STDOUT. (After the first path) (PR #72)
|
||||
- Fix line endings from CRLF to LF for certain project files (PR #67)
|
||||
|
||||
### Changed
|
||||
- Adjust the MD to lower the sensitivity, thus improving the global detection reliability (PR #69 #76)
|
||||
- Allow fallback on specified encoding if any (PR #71)
|
||||
|
||||
## [2.0.3](https://github.com/Ousret/charset_normalizer/compare/2.0.2...2.0.3) (2021-07-16)
|
||||
### Changed
|
||||
- Part of the detection mechanism has been improved to be less sensitive, resulting in more accurate detection results. Especially ASCII. (PR #63)
|
||||
- According to the community wishes, the detection will fall back on ASCII or UTF-8 in a last-resort case. (PR #64)
|
||||
|
||||
## [2.0.2](https://github.com/Ousret/charset_normalizer/compare/2.0.1...2.0.2) (2021-07-15)
|
||||
### Fixed
|
||||
- Empty/Too small JSON payload miss-detection fixed. Report from [@tseaver](https://github.com/tseaver) (PR #59)
|
||||
|
||||
### Changed
|
||||
- Don't inject unicodedata2 into sys.modules from [@akx](https://github.com/akx) (PR #57)
|
||||
|
||||
## [2.0.1](https://github.com/Ousret/charset_normalizer/compare/2.0.0...2.0.1) (2021-07-13)
|
||||
### Fixed
|
||||
- Make it work where there isn't a filesystem available, dropping assets frequencies.json. Report from [@sethmlarson](https://github.com/sethmlarson). (PR #55)
|
||||
- Using explain=False permanently disable the verbose output in the current runtime (PR #47)
|
||||
- One log entry (language target preemptive) was not show in logs when using explain=True (PR #47)
|
||||
- Fix undesired exception (ValueError) on getitem of instance CharsetMatches (PR #52)
|
||||
|
||||
### Changed
|
||||
- Public function normalize default args values were not aligned with from_bytes (PR #53)
|
||||
|
||||
### Added
|
||||
- You may now use charset aliases in cp_isolation and cp_exclusion arguments (PR #47)
|
||||
|
||||
## [2.0.0](https://github.com/Ousret/charset_normalizer/compare/1.4.1...2.0.0) (2021-07-02)
|
||||
### Changed
|
||||
- 4x to 5 times faster than the previous 1.4.0 release. At least 2x faster than Chardet.
|
||||
- Accent has been made on UTF-8 detection, should perform rather instantaneous.
|
||||
- The backward compatibility with Chardet has been greatly improved. The legacy detect function returns an identical charset name whenever possible.
|
||||
- The detection mechanism has been slightly improved, now Turkish content is detected correctly (most of the time)
|
||||
- The program has been rewritten to ease the readability and maintainability. (+Using static typing)+
|
||||
- utf_7 detection has been reinstated.
|
||||
|
||||
### Removed
|
||||
- This package no longer require anything when used with Python 3.5 (Dropped cached_property)
|
||||
- Removed support for these languages: Catalan, Esperanto, Kazakh, Baque, Volapük, Azeri, Galician, Nynorsk, Macedonian, and Serbocroatian.
|
||||
- The exception hook on UnicodeDecodeError has been removed.
|
||||
|
||||
### Deprecated
|
||||
- Methods coherence_non_latin, w_counter, chaos_secondary_pass of the class CharsetMatch are now deprecated and scheduled for removal in v3.0
|
||||
|
||||
### Fixed
|
||||
- The CLI output used the relative path of the file(s). Should be absolute.
|
||||
|
||||
## [1.4.1](https://github.com/Ousret/charset_normalizer/compare/1.4.0...1.4.1) (2021-05-28)
|
||||
### Fixed
|
||||
- Logger configuration/usage no longer conflict with others (PR #44)
|
||||
|
||||
## [1.4.0](https://github.com/Ousret/charset_normalizer/compare/1.3.9...1.4.0) (2021-05-21)
|
||||
### Removed
|
||||
- Using standard logging instead of using the package loguru.
|
||||
- Dropping nose test framework in favor of the maintained pytest.
|
||||
- Choose to not use dragonmapper package to help with gibberish Chinese/CJK text.
|
||||
- Require cached_property only for Python 3.5 due to constraint. Dropping for every other interpreter version.
|
||||
- Stop support for UTF-7 that does not contain a SIG.
|
||||
- Dropping PrettyTable, replaced with pure JSON output in CLI.
|
||||
|
||||
### Fixed
|
||||
- BOM marker in a CharsetNormalizerMatch instance could be False in rare cases even if obviously present. Due to the sub-match factoring process.
|
||||
- Not searching properly for the BOM when trying utf32/16 parent codec.
|
||||
|
||||
### Changed
|
||||
- Improving the package final size by compressing frequencies.json.
|
||||
- Huge improvement over the larges payload.
|
||||
|
||||
### Added
|
||||
- CLI now produces JSON consumable output.
|
||||
- Return ASCII if given sequences fit. Given reasonable confidence.
|
||||
|
||||
## [1.3.9](https://github.com/Ousret/charset_normalizer/compare/1.3.8...1.3.9) (2021-05-13)
|
||||
|
||||
### Fixed
|
||||
- In some very rare cases, you may end up getting encode/decode errors due to a bad bytes payload (PR #40)
|
||||
|
||||
## [1.3.8](https://github.com/Ousret/charset_normalizer/compare/1.3.7...1.3.8) (2021-05-12)
|
||||
|
||||
### Fixed
|
||||
- Empty given payload for detection may cause an exception if trying to access the `alphabets` property. (PR #39)
|
||||
|
||||
## [1.3.7](https://github.com/Ousret/charset_normalizer/compare/1.3.6...1.3.7) (2021-05-12)
|
||||
|
||||
### Fixed
|
||||
- The legacy detect function should return UTF-8-SIG if sig is present in the payload. (PR #38)
|
||||
|
||||
## [1.3.6](https://github.com/Ousret/charset_normalizer/compare/1.3.5...1.3.6) (2021-02-09)
|
||||
|
||||
### Changed
|
||||
- Amend the previous release to allow prettytable 2.0 (PR #35)
|
||||
|
||||
## [1.3.5](https://github.com/Ousret/charset_normalizer/compare/1.3.4...1.3.5) (2021-02-08)
|
||||
|
||||
### Fixed
|
||||
- Fix error while using the package with a python pre-release interpreter (PR #33)
|
||||
|
||||
### Changed
|
||||
- Dependencies refactoring, constraints revised.
|
||||
|
||||
### Added
|
||||
- Add python 3.9 and 3.10 to the supported interpreters
|
||||
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 TAHRI Ahmed R.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,35 @@
|
||||
../../../bin/normalizer,sha256=Weu7f2LxZOAf76g_cM45e9nkon3emmN4QzCBInn8aCU,272
|
||||
charset_normalizer-3.4.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
|
||||
charset_normalizer-3.4.4.dist-info/METADATA,sha256=jVuUFBti8dav19YLvWissTihVdF2ozUY4KKMw7jdkBQ,37303
|
||||
charset_normalizer-3.4.4.dist-info/RECORD,,
|
||||
charset_normalizer-3.4.4.dist-info/WHEEL,sha256=2iHh9e2o6T3nHtu_NVT7Cs7pebIqF94rZK8zrQfgoJI,190
|
||||
charset_normalizer-3.4.4.dist-info/entry_points.txt,sha256=ADSTKrkXZ3hhdOVFi6DcUEHQRS0xfxDIE_pEz4wLIXA,65
|
||||
charset_normalizer-3.4.4.dist-info/licenses/LICENSE,sha256=bQ1Bv-FwrGx9wkjJpj4lTQ-0WmDVCoJX0K-SxuJJuIc,1071
|
||||
charset_normalizer-3.4.4.dist-info/top_level.txt,sha256=7ASyzePr8_xuZWJsnqJjIBtyV8vhEo0wBCv1MPRRi3Q,19
|
||||
charset_normalizer/__init__.py,sha256=OKRxRv2Zhnqk00tqkN0c1BtJjm165fWXLydE52IKuHc,1590
|
||||
charset_normalizer/__main__.py,sha256=yzYxMR-IhKRHYwcSlavEv8oGdwxsR89mr2X09qXGdps,109
|
||||
charset_normalizer/__pycache__/__init__.cpython-313.pyc,,
|
||||
charset_normalizer/__pycache__/__main__.cpython-313.pyc,,
|
||||
charset_normalizer/__pycache__/api.cpython-313.pyc,,
|
||||
charset_normalizer/__pycache__/cd.cpython-313.pyc,,
|
||||
charset_normalizer/__pycache__/constant.cpython-313.pyc,,
|
||||
charset_normalizer/__pycache__/legacy.cpython-313.pyc,,
|
||||
charset_normalizer/__pycache__/md.cpython-313.pyc,,
|
||||
charset_normalizer/__pycache__/models.cpython-313.pyc,,
|
||||
charset_normalizer/__pycache__/utils.cpython-313.pyc,,
|
||||
charset_normalizer/__pycache__/version.cpython-313.pyc,,
|
||||
charset_normalizer/api.py,sha256=V07i8aVeCD8T2fSia3C-fn0i9t8qQguEBhsqszg32Ns,22668
|
||||
charset_normalizer/cd.py,sha256=WKTo1HDb-H9HfCDc3Bfwq5jzS25Ziy9SE2a74SgTq88,12522
|
||||
charset_normalizer/cli/__init__.py,sha256=D8I86lFk2-py45JvqxniTirSj_sFyE6sjaY_0-G1shc,136
|
||||
charset_normalizer/cli/__main__.py,sha256=dMaXG6IJXRvqq8z2tig7Qb83-BpWTln55ooiku5_uvg,12646
|
||||
charset_normalizer/cli/__pycache__/__init__.cpython-313.pyc,,
|
||||
charset_normalizer/cli/__pycache__/__main__.cpython-313.pyc,,
|
||||
charset_normalizer/constant.py,sha256=7UVY4ldYhmQMHUdgQ_sgZmzcQ0xxYxpBunqSZ-XJZ8U,42713
|
||||
charset_normalizer/legacy.py,sha256=sYBzSpzsRrg_wF4LP536pG64BItw7Tqtc3SMQAHvFLM,2731
|
||||
charset_normalizer/md.cpython-313-x86_64-linux-gnu.so,sha256=sZ7umtJLjKfA83NFJ7npkiDyr06zDT8cWtl6uIx2MsM,15912
|
||||
charset_normalizer/md.py,sha256=-_oN3h3_X99nkFfqamD3yu45DC_wfk5odH0Tr_CQiXs,20145
|
||||
charset_normalizer/md__mypyc.cpython-313-x86_64-linux-gnu.so,sha256=i-yavqPJtZwjTKvP9hBLZ8CLZD88rVtguaSoLHso_Oc,291056
|
||||
charset_normalizer/models.py,sha256=lKXhOnIPtiakbK3i__J9wpOfzx3JDTKj7Dn3Rg0VaRI,12394
|
||||
charset_normalizer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||
charset_normalizer/utils.py,sha256=sTejPgrdlNsKNucZfJCxJ95lMTLA0ShHLLE3n5wpT9Q,12170
|
||||
charset_normalizer/version.py,sha256=nKE4qBNk5WA4LIJ_yIH_aSDfvtsyizkWMg-PUG-UZVk,115
|
||||
@@ -0,0 +1,7 @@
|
||||
Wheel-Version: 1.0
|
||||
Generator: setuptools (80.9.0)
|
||||
Root-Is-Purelib: false
|
||||
Tag: cp313-cp313-manylinux_2_17_x86_64
|
||||
Tag: cp313-cp313-manylinux2014_x86_64
|
||||
Tag: cp313-cp313-manylinux_2_28_x86_64
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
[console_scripts]
|
||||
normalizer = charset_normalizer.cli:cli_detect
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 TAHRI Ahmed R.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1 @@
|
||||
charset_normalizer
|
||||
@@ -0,0 +1,48 @@
|
||||
"""
|
||||
Charset-Normalizer
|
||||
~~~~~~~~~~~~~~
|
||||
The Real First Universal Charset Detector.
|
||||
A library that helps you read text from an unknown charset encoding.
|
||||
Motivated by chardet, This package is trying to resolve the issue by taking a new approach.
|
||||
All IANA character set names for which the Python core library provides codecs are supported.
|
||||
|
||||
Basic usage:
|
||||
>>> from charset_normalizer import from_bytes
|
||||
>>> results = from_bytes('Bсеки човек има право на образование. Oбразованието!'.encode('utf_8'))
|
||||
>>> best_guess = results.best()
|
||||
>>> str(best_guess)
|
||||
'Bсеки човек има право на образование. Oбразованието!'
|
||||
|
||||
Others methods and usages are available - see the full documentation
|
||||
at <https://github.com/Ousret/charset_normalizer>.
|
||||
:copyright: (c) 2021 by Ahmed TAHRI
|
||||
:license: MIT, see LICENSE for more details.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from .api import from_bytes, from_fp, from_path, is_binary
|
||||
from .legacy import detect
|
||||
from .models import CharsetMatch, CharsetMatches
|
||||
from .utils import set_logging_handler
|
||||
from .version import VERSION, __version__
|
||||
|
||||
__all__ = (
|
||||
"from_fp",
|
||||
"from_path",
|
||||
"from_bytes",
|
||||
"is_binary",
|
||||
"detect",
|
||||
"CharsetMatch",
|
||||
"CharsetMatches",
|
||||
"__version__",
|
||||
"VERSION",
|
||||
"set_logging_handler",
|
||||
)
|
||||
|
||||
# Attach a NullHandler to the top level logger by default
|
||||
# https://docs.python.org/3.3/howto/logging.html#configuring-logging-for-a-library
|
||||
|
||||
logging.getLogger("charset_normalizer").addHandler(logging.NullHandler())
|
||||
@@ -0,0 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .cli import cli_detect
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli_detect()
|
||||
@@ -0,0 +1,669 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from os import PathLike
|
||||
from typing import BinaryIO
|
||||
|
||||
from .cd import (
|
||||
coherence_ratio,
|
||||
encoding_languages,
|
||||
mb_encoding_languages,
|
||||
merge_coherence_ratios,
|
||||
)
|
||||
from .constant import IANA_SUPPORTED, TOO_BIG_SEQUENCE, TOO_SMALL_SEQUENCE, TRACE
|
||||
from .md import mess_ratio
|
||||
from .models import CharsetMatch, CharsetMatches
|
||||
from .utils import (
|
||||
any_specified_encoding,
|
||||
cut_sequence_chunks,
|
||||
iana_name,
|
||||
identify_sig_or_bom,
|
||||
is_cp_similar,
|
||||
is_multi_byte_encoding,
|
||||
should_strip_sig_or_bom,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("charset_normalizer")
|
||||
explain_handler = logging.StreamHandler()
|
||||
explain_handler.setFormatter(
|
||||
logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
|
||||
)
|
||||
|
||||
|
||||
def from_bytes(
|
||||
sequences: bytes | bytearray,
|
||||
steps: int = 5,
|
||||
chunk_size: int = 512,
|
||||
threshold: float = 0.2,
|
||||
cp_isolation: list[str] | None = None,
|
||||
cp_exclusion: list[str] | None = None,
|
||||
preemptive_behaviour: bool = True,
|
||||
explain: bool = False,
|
||||
language_threshold: float = 0.1,
|
||||
enable_fallback: bool = True,
|
||||
) -> CharsetMatches:
|
||||
"""
|
||||
Given a raw bytes sequence, return the best possibles charset usable to render str objects.
|
||||
If there is no results, it is a strong indicator that the source is binary/not text.
|
||||
By default, the process will extract 5 blocks of 512o each to assess the mess and coherence of a given sequence.
|
||||
And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will.
|
||||
|
||||
The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page
|
||||
but never take it for granted. Can improve the performance.
|
||||
|
||||
You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that
|
||||
purpose.
|
||||
|
||||
This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32.
|
||||
By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain'
|
||||
toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging.
|
||||
Custom logging format and handler can be set manually.
|
||||
"""
|
||||
|
||||
if not isinstance(sequences, (bytearray, bytes)):
|
||||
raise TypeError(
|
||||
"Expected object of type bytes or bytearray, got: {}".format(
|
||||
type(sequences)
|
||||
)
|
||||
)
|
||||
|
||||
if explain:
|
||||
previous_logger_level: int = logger.level
|
||||
logger.addHandler(explain_handler)
|
||||
logger.setLevel(TRACE)
|
||||
|
||||
length: int = len(sequences)
|
||||
|
||||
if length == 0:
|
||||
logger.debug("Encoding detection on empty bytes, assuming utf_8 intention.")
|
||||
if explain: # Defensive: ensure exit path clean handler
|
||||
logger.removeHandler(explain_handler)
|
||||
logger.setLevel(previous_logger_level or logging.WARNING)
|
||||
return CharsetMatches([CharsetMatch(sequences, "utf_8", 0.0, False, [], "")])
|
||||
|
||||
if cp_isolation is not None:
|
||||
logger.log(
|
||||
TRACE,
|
||||
"cp_isolation is set. use this flag for debugging purpose. "
|
||||
"limited list of encoding allowed : %s.",
|
||||
", ".join(cp_isolation),
|
||||
)
|
||||
cp_isolation = [iana_name(cp, False) for cp in cp_isolation]
|
||||
else:
|
||||
cp_isolation = []
|
||||
|
||||
if cp_exclusion is not None:
|
||||
logger.log(
|
||||
TRACE,
|
||||
"cp_exclusion is set. use this flag for debugging purpose. "
|
||||
"limited list of encoding excluded : %s.",
|
||||
", ".join(cp_exclusion),
|
||||
)
|
||||
cp_exclusion = [iana_name(cp, False) for cp in cp_exclusion]
|
||||
else:
|
||||
cp_exclusion = []
|
||||
|
||||
if length <= (chunk_size * steps):
|
||||
logger.log(
|
||||
TRACE,
|
||||
"override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.",
|
||||
steps,
|
||||
chunk_size,
|
||||
length,
|
||||
)
|
||||
steps = 1
|
||||
chunk_size = length
|
||||
|
||||
if steps > 1 and length / steps < chunk_size:
|
||||
chunk_size = int(length / steps)
|
||||
|
||||
is_too_small_sequence: bool = len(sequences) < TOO_SMALL_SEQUENCE
|
||||
is_too_large_sequence: bool = len(sequences) >= TOO_BIG_SEQUENCE
|
||||
|
||||
if is_too_small_sequence:
|
||||
logger.log(
|
||||
TRACE,
|
||||
"Trying to detect encoding from a tiny portion of ({}) byte(s).".format(
|
||||
length
|
||||
),
|
||||
)
|
||||
elif is_too_large_sequence:
|
||||
logger.log(
|
||||
TRACE,
|
||||
"Using lazy str decoding because the payload is quite large, ({}) byte(s).".format(
|
||||
length
|
||||
),
|
||||
)
|
||||
|
||||
prioritized_encodings: list[str] = []
|
||||
|
||||
specified_encoding: str | None = (
|
||||
any_specified_encoding(sequences) if preemptive_behaviour else None
|
||||
)
|
||||
|
||||
if specified_encoding is not None:
|
||||
prioritized_encodings.append(specified_encoding)
|
||||
logger.log(
|
||||
TRACE,
|
||||
"Detected declarative mark in sequence. Priority +1 given for %s.",
|
||||
specified_encoding,
|
||||
)
|
||||
|
||||
tested: set[str] = set()
|
||||
tested_but_hard_failure: list[str] = []
|
||||
tested_but_soft_failure: list[str] = []
|
||||
|
||||
fallback_ascii: CharsetMatch | None = None
|
||||
fallback_u8: CharsetMatch | None = None
|
||||
fallback_specified: CharsetMatch | None = None
|
||||
|
||||
results: CharsetMatches = CharsetMatches()
|
||||
|
||||
early_stop_results: CharsetMatches = CharsetMatches()
|
||||
|
||||
sig_encoding, sig_payload = identify_sig_or_bom(sequences)
|
||||
|
||||
if sig_encoding is not None:
|
||||
prioritized_encodings.append(sig_encoding)
|
||||
logger.log(
|
||||
TRACE,
|
||||
"Detected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.",
|
||||
len(sig_payload),
|
||||
sig_encoding,
|
||||
)
|
||||
|
||||
prioritized_encodings.append("ascii")
|
||||
|
||||
if "utf_8" not in prioritized_encodings:
|
||||
prioritized_encodings.append("utf_8")
|
||||
|
||||
for encoding_iana in prioritized_encodings + IANA_SUPPORTED:
|
||||
if cp_isolation and encoding_iana not in cp_isolation:
|
||||
continue
|
||||
|
||||
if cp_exclusion and encoding_iana in cp_exclusion:
|
||||
continue
|
||||
|
||||
if encoding_iana in tested:
|
||||
continue
|
||||
|
||||
tested.add(encoding_iana)
|
||||
|
||||
decoded_payload: str | None = None
|
||||
bom_or_sig_available: bool = sig_encoding == encoding_iana
|
||||
strip_sig_or_bom: bool = bom_or_sig_available and should_strip_sig_or_bom(
|
||||
encoding_iana
|
||||
)
|
||||
|
||||
if encoding_iana in {"utf_16", "utf_32"} and not bom_or_sig_available:
|
||||
logger.log(
|
||||
TRACE,
|
||||
"Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.",
|
||||
encoding_iana,
|
||||
)
|
||||
continue
|
||||
if encoding_iana in {"utf_7"} and not bom_or_sig_available:
|
||||
logger.log(
|
||||
TRACE,
|
||||
"Encoding %s won't be tested as-is because detection is unreliable without BOM/SIG.",
|
||||
encoding_iana,
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
is_multi_byte_decoder: bool = is_multi_byte_encoding(encoding_iana)
|
||||
except (ModuleNotFoundError, ImportError):
|
||||
logger.log(
|
||||
TRACE,
|
||||
"Encoding %s does not provide an IncrementalDecoder",
|
||||
encoding_iana,
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
if is_too_large_sequence and is_multi_byte_decoder is False:
|
||||
str(
|
||||
(
|
||||
sequences[: int(50e4)]
|
||||
if strip_sig_or_bom is False
|
||||
else sequences[len(sig_payload) : int(50e4)]
|
||||
),
|
||||
encoding=encoding_iana,
|
||||
)
|
||||
else:
|
||||
decoded_payload = str(
|
||||
(
|
||||
sequences
|
||||
if strip_sig_or_bom is False
|
||||
else sequences[len(sig_payload) :]
|
||||
),
|
||||
encoding=encoding_iana,
|
||||
)
|
||||
except (UnicodeDecodeError, LookupError) as e:
|
||||
if not isinstance(e, LookupError):
|
||||
logger.log(
|
||||
TRACE,
|
||||
"Code page %s does not fit given bytes sequence at ALL. %s",
|
||||
encoding_iana,
|
||||
str(e),
|
||||
)
|
||||
tested_but_hard_failure.append(encoding_iana)
|
||||
continue
|
||||
|
||||
similar_soft_failure_test: bool = False
|
||||
|
||||
for encoding_soft_failed in tested_but_soft_failure:
|
||||
if is_cp_similar(encoding_iana, encoding_soft_failed):
|
||||
similar_soft_failure_test = True
|
||||
break
|
||||
|
||||
if similar_soft_failure_test:
|
||||
logger.log(
|
||||
TRACE,
|
||||
"%s is deemed too similar to code page %s and was consider unsuited already. Continuing!",
|
||||
encoding_iana,
|
||||
encoding_soft_failed,
|
||||
)
|
||||
continue
|
||||
|
||||
r_ = range(
|
||||
0 if not bom_or_sig_available else len(sig_payload),
|
||||
length,
|
||||
int(length / steps),
|
||||
)
|
||||
|
||||
multi_byte_bonus: bool = (
|
||||
is_multi_byte_decoder
|
||||
and decoded_payload is not None
|
||||
and len(decoded_payload) < length
|
||||
)
|
||||
|
||||
if multi_byte_bonus:
|
||||
logger.log(
|
||||
TRACE,
|
||||
"Code page %s is a multi byte encoding table and it appear that at least one character "
|
||||
"was encoded using n-bytes.",
|
||||
encoding_iana,
|
||||
)
|
||||
|
||||
max_chunk_gave_up: int = int(len(r_) / 4)
|
||||
|
||||
max_chunk_gave_up = max(max_chunk_gave_up, 2)
|
||||
early_stop_count: int = 0
|
||||
lazy_str_hard_failure = False
|
||||
|
||||
md_chunks: list[str] = []
|
||||
md_ratios = []
|
||||
|
||||
try:
|
||||
for chunk in cut_sequence_chunks(
|
||||
sequences,
|
||||
encoding_iana,
|
||||
r_,
|
||||
chunk_size,
|
||||
bom_or_sig_available,
|
||||
strip_sig_or_bom,
|
||||
sig_payload,
|
||||
is_multi_byte_decoder,
|
||||
decoded_payload,
|
||||
):
|
||||
md_chunks.append(chunk)
|
||||
|
||||
md_ratios.append(
|
||||
mess_ratio(
|
||||
chunk,
|
||||
threshold,
|
||||
explain is True and 1 <= len(cp_isolation) <= 2,
|
||||
)
|
||||
)
|
||||
|
||||
if md_ratios[-1] >= threshold:
|
||||
early_stop_count += 1
|
||||
|
||||
if (early_stop_count >= max_chunk_gave_up) or (
|
||||
bom_or_sig_available and strip_sig_or_bom is False
|
||||
):
|
||||
break
|
||||
except (
|
||||
UnicodeDecodeError
|
||||
) as e: # Lazy str loading may have missed something there
|
||||
logger.log(
|
||||
TRACE,
|
||||
"LazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %s",
|
||||
encoding_iana,
|
||||
str(e),
|
||||
)
|
||||
early_stop_count = max_chunk_gave_up
|
||||
lazy_str_hard_failure = True
|
||||
|
||||
# We might want to check the sequence again with the whole content
|
||||
# Only if initial MD tests passes
|
||||
if (
|
||||
not lazy_str_hard_failure
|
||||
and is_too_large_sequence
|
||||
and not is_multi_byte_decoder
|
||||
):
|
||||
try:
|
||||
sequences[int(50e3) :].decode(encoding_iana, errors="strict")
|
||||
except UnicodeDecodeError as e:
|
||||
logger.log(
|
||||
TRACE,
|
||||
"LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %s",
|
||||
encoding_iana,
|
||||
str(e),
|
||||
)
|
||||
tested_but_hard_failure.append(encoding_iana)
|
||||
continue
|
||||
|
||||
mean_mess_ratio: float = sum(md_ratios) / len(md_ratios) if md_ratios else 0.0
|
||||
if mean_mess_ratio >= threshold or early_stop_count >= max_chunk_gave_up:
|
||||
tested_but_soft_failure.append(encoding_iana)
|
||||
logger.log(
|
||||
TRACE,
|
||||
"%s was excluded because of initial chaos probing. Gave up %i time(s). "
|
||||
"Computed mean chaos is %f %%.",
|
||||
encoding_iana,
|
||||
early_stop_count,
|
||||
round(mean_mess_ratio * 100, ndigits=3),
|
||||
)
|
||||
# Preparing those fallbacks in case we got nothing.
|
||||
if (
|
||||
enable_fallback
|
||||
and encoding_iana
|
||||
in ["ascii", "utf_8", specified_encoding, "utf_16", "utf_32"]
|
||||
and not lazy_str_hard_failure
|
||||
):
|
||||
fallback_entry = CharsetMatch(
|
||||
sequences,
|
||||
encoding_iana,
|
||||
threshold,
|
||||
bom_or_sig_available,
|
||||
[],
|
||||
decoded_payload,
|
||||
preemptive_declaration=specified_encoding,
|
||||
)
|
||||
if encoding_iana == specified_encoding:
|
||||
fallback_specified = fallback_entry
|
||||
elif encoding_iana == "ascii":
|
||||
fallback_ascii = fallback_entry
|
||||
else:
|
||||
fallback_u8 = fallback_entry
|
||||
continue
|
||||
|
||||
logger.log(
|
||||
TRACE,
|
||||
"%s passed initial chaos probing. Mean measured chaos is %f %%",
|
||||
encoding_iana,
|
||||
round(mean_mess_ratio * 100, ndigits=3),
|
||||
)
|
||||
|
||||
if not is_multi_byte_decoder:
|
||||
target_languages: list[str] = encoding_languages(encoding_iana)
|
||||
else:
|
||||
target_languages = mb_encoding_languages(encoding_iana)
|
||||
|
||||
if target_languages:
|
||||
logger.log(
|
||||
TRACE,
|
||||
"{} should target any language(s) of {}".format(
|
||||
encoding_iana, str(target_languages)
|
||||
),
|
||||
)
|
||||
|
||||
cd_ratios = []
|
||||
|
||||
# We shall skip the CD when its about ASCII
|
||||
# Most of the time its not relevant to run "language-detection" on it.
|
||||
if encoding_iana != "ascii":
|
||||
for chunk in md_chunks:
|
||||
chunk_languages = coherence_ratio(
|
||||
chunk,
|
||||
language_threshold,
|
||||
",".join(target_languages) if target_languages else None,
|
||||
)
|
||||
|
||||
cd_ratios.append(chunk_languages)
|
||||
|
||||
cd_ratios_merged = merge_coherence_ratios(cd_ratios)
|
||||
|
||||
if cd_ratios_merged:
|
||||
logger.log(
|
||||
TRACE,
|
||||
"We detected language {} using {}".format(
|
||||
cd_ratios_merged, encoding_iana
|
||||
),
|
||||
)
|
||||
|
||||
current_match = CharsetMatch(
|
||||
sequences,
|
||||
encoding_iana,
|
||||
mean_mess_ratio,
|
||||
bom_or_sig_available,
|
||||
cd_ratios_merged,
|
||||
(
|
||||
decoded_payload
|
||||
if (
|
||||
is_too_large_sequence is False
|
||||
or encoding_iana in [specified_encoding, "ascii", "utf_8"]
|
||||
)
|
||||
else None
|
||||
),
|
||||
preemptive_declaration=specified_encoding,
|
||||
)
|
||||
|
||||
results.append(current_match)
|
||||
|
||||
if (
|
||||
encoding_iana in [specified_encoding, "ascii", "utf_8"]
|
||||
and mean_mess_ratio < 0.1
|
||||
):
|
||||
# If md says nothing to worry about, then... stop immediately!
|
||||
if mean_mess_ratio == 0.0:
|
||||
logger.debug(
|
||||
"Encoding detection: %s is most likely the one.",
|
||||
current_match.encoding,
|
||||
)
|
||||
if explain: # Defensive: ensure exit path clean handler
|
||||
logger.removeHandler(explain_handler)
|
||||
logger.setLevel(previous_logger_level)
|
||||
return CharsetMatches([current_match])
|
||||
|
||||
early_stop_results.append(current_match)
|
||||
|
||||
if (
|
||||
len(early_stop_results)
|
||||
and (specified_encoding is None or specified_encoding in tested)
|
||||
and "ascii" in tested
|
||||
and "utf_8" in tested
|
||||
):
|
||||
probable_result: CharsetMatch = early_stop_results.best() # type: ignore[assignment]
|
||||
logger.debug(
|
||||
"Encoding detection: %s is most likely the one.",
|
||||
probable_result.encoding,
|
||||
)
|
||||
if explain: # Defensive: ensure exit path clean handler
|
||||
logger.removeHandler(explain_handler)
|
||||
logger.setLevel(previous_logger_level)
|
||||
|
||||
return CharsetMatches([probable_result])
|
||||
|
||||
if encoding_iana == sig_encoding:
|
||||
logger.debug(
|
||||
"Encoding detection: %s is most likely the one as we detected a BOM or SIG within "
|
||||
"the beginning of the sequence.",
|
||||
encoding_iana,
|
||||
)
|
||||
if explain: # Defensive: ensure exit path clean handler
|
||||
logger.removeHandler(explain_handler)
|
||||
logger.setLevel(previous_logger_level)
|
||||
return CharsetMatches([results[encoding_iana]])
|
||||
|
||||
if len(results) == 0:
|
||||
if fallback_u8 or fallback_ascii or fallback_specified:
|
||||
logger.log(
|
||||
TRACE,
|
||||
"Nothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.",
|
||||
)
|
||||
|
||||
if fallback_specified:
|
||||
logger.debug(
|
||||
"Encoding detection: %s will be used as a fallback match",
|
||||
fallback_specified.encoding,
|
||||
)
|
||||
results.append(fallback_specified)
|
||||
elif (
|
||||
(fallback_u8 and fallback_ascii is None)
|
||||
or (
|
||||
fallback_u8
|
||||
and fallback_ascii
|
||||
and fallback_u8.fingerprint != fallback_ascii.fingerprint
|
||||
)
|
||||
or (fallback_u8 is not None)
|
||||
):
|
||||
logger.debug("Encoding detection: utf_8 will be used as a fallback match")
|
||||
results.append(fallback_u8)
|
||||
elif fallback_ascii:
|
||||
logger.debug("Encoding detection: ascii will be used as a fallback match")
|
||||
results.append(fallback_ascii)
|
||||
|
||||
if results:
|
||||
logger.debug(
|
||||
"Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.",
|
||||
results.best().encoding, # type: ignore
|
||||
len(results) - 1,
|
||||
)
|
||||
else:
|
||||
logger.debug("Encoding detection: Unable to determine any suitable charset.")
|
||||
|
||||
if explain:
|
||||
logger.removeHandler(explain_handler)
|
||||
logger.setLevel(previous_logger_level)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def from_fp(
|
||||
fp: BinaryIO,
|
||||
steps: int = 5,
|
||||
chunk_size: int = 512,
|
||||
threshold: float = 0.20,
|
||||
cp_isolation: list[str] | None = None,
|
||||
cp_exclusion: list[str] | None = None,
|
||||
preemptive_behaviour: bool = True,
|
||||
explain: bool = False,
|
||||
language_threshold: float = 0.1,
|
||||
enable_fallback: bool = True,
|
||||
) -> CharsetMatches:
|
||||
"""
|
||||
Same thing than the function from_bytes but using a file pointer that is already ready.
|
||||
Will not close the file pointer.
|
||||
"""
|
||||
return from_bytes(
|
||||
fp.read(),
|
||||
steps,
|
||||
chunk_size,
|
||||
threshold,
|
||||
cp_isolation,
|
||||
cp_exclusion,
|
||||
preemptive_behaviour,
|
||||
explain,
|
||||
language_threshold,
|
||||
enable_fallback,
|
||||
)
|
||||
|
||||
|
||||
def from_path(
|
||||
path: str | bytes | PathLike, # type: ignore[type-arg]
|
||||
steps: int = 5,
|
||||
chunk_size: int = 512,
|
||||
threshold: float = 0.20,
|
||||
cp_isolation: list[str] | None = None,
|
||||
cp_exclusion: list[str] | None = None,
|
||||
preemptive_behaviour: bool = True,
|
||||
explain: bool = False,
|
||||
language_threshold: float = 0.1,
|
||||
enable_fallback: bool = True,
|
||||
) -> CharsetMatches:
|
||||
"""
|
||||
Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode.
|
||||
Can raise IOError.
|
||||
"""
|
||||
with open(path, "rb") as fp:
|
||||
return from_fp(
|
||||
fp,
|
||||
steps,
|
||||
chunk_size,
|
||||
threshold,
|
||||
cp_isolation,
|
||||
cp_exclusion,
|
||||
preemptive_behaviour,
|
||||
explain,
|
||||
language_threshold,
|
||||
enable_fallback,
|
||||
)
|
||||
|
||||
|
||||
def is_binary(
|
||||
fp_or_path_or_payload: PathLike | str | BinaryIO | bytes, # type: ignore[type-arg]
|
||||
steps: int = 5,
|
||||
chunk_size: int = 512,
|
||||
threshold: float = 0.20,
|
||||
cp_isolation: list[str] | None = None,
|
||||
cp_exclusion: list[str] | None = None,
|
||||
preemptive_behaviour: bool = True,
|
||||
explain: bool = False,
|
||||
language_threshold: float = 0.1,
|
||||
enable_fallback: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string.
|
||||
Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match
|
||||
are disabled to be stricter around ASCII-compatible but unlikely to be a string.
|
||||
"""
|
||||
if isinstance(fp_or_path_or_payload, (str, PathLike)):
|
||||
guesses = from_path(
|
||||
fp_or_path_or_payload,
|
||||
steps=steps,
|
||||
chunk_size=chunk_size,
|
||||
threshold=threshold,
|
||||
cp_isolation=cp_isolation,
|
||||
cp_exclusion=cp_exclusion,
|
||||
preemptive_behaviour=preemptive_behaviour,
|
||||
explain=explain,
|
||||
language_threshold=language_threshold,
|
||||
enable_fallback=enable_fallback,
|
||||
)
|
||||
elif isinstance(
|
||||
fp_or_path_or_payload,
|
||||
(
|
||||
bytes,
|
||||
bytearray,
|
||||
),
|
||||
):
|
||||
guesses = from_bytes(
|
||||
fp_or_path_or_payload,
|
||||
steps=steps,
|
||||
chunk_size=chunk_size,
|
||||
threshold=threshold,
|
||||
cp_isolation=cp_isolation,
|
||||
cp_exclusion=cp_exclusion,
|
||||
preemptive_behaviour=preemptive_behaviour,
|
||||
explain=explain,
|
||||
language_threshold=language_threshold,
|
||||
enable_fallback=enable_fallback,
|
||||
)
|
||||
else:
|
||||
guesses = from_fp(
|
||||
fp_or_path_or_payload,
|
||||
steps=steps,
|
||||
chunk_size=chunk_size,
|
||||
threshold=threshold,
|
||||
cp_isolation=cp_isolation,
|
||||
cp_exclusion=cp_exclusion,
|
||||
preemptive_behaviour=preemptive_behaviour,
|
||||
explain=explain,
|
||||
language_threshold=language_threshold,
|
||||
enable_fallback=enable_fallback,
|
||||
)
|
||||
|
||||
return not guesses
|
||||
@@ -0,0 +1,395 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from codecs import IncrementalDecoder
|
||||
from collections import Counter
|
||||
from functools import lru_cache
|
||||
from typing import Counter as TypeCounter
|
||||
|
||||
from .constant import (
|
||||
FREQUENCIES,
|
||||
KO_NAMES,
|
||||
LANGUAGE_SUPPORTED_COUNT,
|
||||
TOO_SMALL_SEQUENCE,
|
||||
ZH_NAMES,
|
||||
)
|
||||
from .md import is_suspiciously_successive_range
|
||||
from .models import CoherenceMatches
|
||||
from .utils import (
|
||||
is_accentuated,
|
||||
is_latin,
|
||||
is_multi_byte_encoding,
|
||||
is_unicode_range_secondary,
|
||||
unicode_range,
|
||||
)
|
||||
|
||||
|
||||
def encoding_unicode_range(iana_name: str) -> list[str]:
|
||||
"""
|
||||
Return associated unicode ranges in a single byte code page.
|
||||
"""
|
||||
if is_multi_byte_encoding(iana_name):
|
||||
raise OSError("Function not supported on multi-byte code page")
|
||||
|
||||
decoder = importlib.import_module(f"encodings.{iana_name}").IncrementalDecoder
|
||||
|
||||
p: IncrementalDecoder = decoder(errors="ignore")
|
||||
seen_ranges: dict[str, int] = {}
|
||||
character_count: int = 0
|
||||
|
||||
for i in range(0x40, 0xFF):
|
||||
chunk: str = p.decode(bytes([i]))
|
||||
|
||||
if chunk:
|
||||
character_range: str | None = unicode_range(chunk)
|
||||
|
||||
if character_range is None:
|
||||
continue
|
||||
|
||||
if is_unicode_range_secondary(character_range) is False:
|
||||
if character_range not in seen_ranges:
|
||||
seen_ranges[character_range] = 0
|
||||
seen_ranges[character_range] += 1
|
||||
character_count += 1
|
||||
|
||||
return sorted(
|
||||
[
|
||||
character_range
|
||||
for character_range in seen_ranges
|
||||
if seen_ranges[character_range] / character_count >= 0.15
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def unicode_range_languages(primary_range: str) -> list[str]:
|
||||
"""
|
||||
Return inferred languages used with a unicode range.
|
||||
"""
|
||||
languages: list[str] = []
|
||||
|
||||
for language, characters in FREQUENCIES.items():
|
||||
for character in characters:
|
||||
if unicode_range(character) == primary_range:
|
||||
languages.append(language)
|
||||
break
|
||||
|
||||
return languages
|
||||
|
||||
|
||||
@lru_cache()
|
||||
def encoding_languages(iana_name: str) -> list[str]:
|
||||
"""
|
||||
Single-byte encoding language association. Some code page are heavily linked to particular language(s).
|
||||
This function does the correspondence.
|
||||
"""
|
||||
unicode_ranges: list[str] = encoding_unicode_range(iana_name)
|
||||
primary_range: str | None = None
|
||||
|
||||
for specified_range in unicode_ranges:
|
||||
if "Latin" not in specified_range:
|
||||
primary_range = specified_range
|
||||
break
|
||||
|
||||
if primary_range is None:
|
||||
return ["Latin Based"]
|
||||
|
||||
return unicode_range_languages(primary_range)
|
||||
|
||||
|
||||
@lru_cache()
|
||||
def mb_encoding_languages(iana_name: str) -> list[str]:
|
||||
"""
|
||||
Multi-byte encoding language association. Some code page are heavily linked to particular language(s).
|
||||
This function does the correspondence.
|
||||
"""
|
||||
if (
|
||||
iana_name.startswith("shift_")
|
||||
or iana_name.startswith("iso2022_jp")
|
||||
or iana_name.startswith("euc_j")
|
||||
or iana_name == "cp932"
|
||||
):
|
||||
return ["Japanese"]
|
||||
if iana_name.startswith("gb") or iana_name in ZH_NAMES:
|
||||
return ["Chinese"]
|
||||
if iana_name.startswith("iso2022_kr") or iana_name in KO_NAMES:
|
||||
return ["Korean"]
|
||||
|
||||
return []
|
||||
|
||||
|
||||
@lru_cache(maxsize=LANGUAGE_SUPPORTED_COUNT)
|
||||
def get_target_features(language: str) -> tuple[bool, bool]:
|
||||
"""
|
||||
Determine main aspects from a supported language if it contains accents and if is pure Latin.
|
||||
"""
|
||||
target_have_accents: bool = False
|
||||
target_pure_latin: bool = True
|
||||
|
||||
for character in FREQUENCIES[language]:
|
||||
if not target_have_accents and is_accentuated(character):
|
||||
target_have_accents = True
|
||||
if target_pure_latin and is_latin(character) is False:
|
||||
target_pure_latin = False
|
||||
|
||||
return target_have_accents, target_pure_latin
|
||||
|
||||
|
||||
def alphabet_languages(
|
||||
characters: list[str], ignore_non_latin: bool = False
|
||||
) -> list[str]:
|
||||
"""
|
||||
Return associated languages associated to given characters.
|
||||
"""
|
||||
languages: list[tuple[str, float]] = []
|
||||
|
||||
source_have_accents = any(is_accentuated(character) for character in characters)
|
||||
|
||||
for language, language_characters in FREQUENCIES.items():
|
||||
target_have_accents, target_pure_latin = get_target_features(language)
|
||||
|
||||
if ignore_non_latin and target_pure_latin is False:
|
||||
continue
|
||||
|
||||
if target_have_accents is False and source_have_accents:
|
||||
continue
|
||||
|
||||
character_count: int = len(language_characters)
|
||||
|
||||
character_match_count: int = len(
|
||||
[c for c in language_characters if c in characters]
|
||||
)
|
||||
|
||||
ratio: float = character_match_count / character_count
|
||||
|
||||
if ratio >= 0.2:
|
||||
languages.append((language, ratio))
|
||||
|
||||
languages = sorted(languages, key=lambda x: x[1], reverse=True)
|
||||
|
||||
return [compatible_language[0] for compatible_language in languages]
|
||||
|
||||
|
||||
def characters_popularity_compare(
|
||||
language: str, ordered_characters: list[str]
|
||||
) -> float:
|
||||
"""
|
||||
Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language.
|
||||
The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit).
|
||||
Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.)
|
||||
"""
|
||||
if language not in FREQUENCIES:
|
||||
raise ValueError(f"{language} not available")
|
||||
|
||||
character_approved_count: int = 0
|
||||
FREQUENCIES_language_set = set(FREQUENCIES[language])
|
||||
|
||||
ordered_characters_count: int = len(ordered_characters)
|
||||
target_language_characters_count: int = len(FREQUENCIES[language])
|
||||
|
||||
large_alphabet: bool = target_language_characters_count > 26
|
||||
|
||||
for character, character_rank in zip(
|
||||
ordered_characters, range(0, ordered_characters_count)
|
||||
):
|
||||
if character not in FREQUENCIES_language_set:
|
||||
continue
|
||||
|
||||
character_rank_in_language: int = FREQUENCIES[language].index(character)
|
||||
expected_projection_ratio: float = (
|
||||
target_language_characters_count / ordered_characters_count
|
||||
)
|
||||
character_rank_projection: int = int(character_rank * expected_projection_ratio)
|
||||
|
||||
if (
|
||||
large_alphabet is False
|
||||
and abs(character_rank_projection - character_rank_in_language) > 4
|
||||
):
|
||||
continue
|
||||
|
||||
if (
|
||||
large_alphabet is True
|
||||
and abs(character_rank_projection - character_rank_in_language)
|
||||
< target_language_characters_count / 3
|
||||
):
|
||||
character_approved_count += 1
|
||||
continue
|
||||
|
||||
characters_before_source: list[str] = FREQUENCIES[language][
|
||||
0:character_rank_in_language
|
||||
]
|
||||
characters_after_source: list[str] = FREQUENCIES[language][
|
||||
character_rank_in_language:
|
||||
]
|
||||
characters_before: list[str] = ordered_characters[0:character_rank]
|
||||
characters_after: list[str] = ordered_characters[character_rank:]
|
||||
|
||||
before_match_count: int = len(
|
||||
set(characters_before) & set(characters_before_source)
|
||||
)
|
||||
|
||||
after_match_count: int = len(
|
||||
set(characters_after) & set(characters_after_source)
|
||||
)
|
||||
|
||||
if len(characters_before_source) == 0 and before_match_count <= 4:
|
||||
character_approved_count += 1
|
||||
continue
|
||||
|
||||
if len(characters_after_source) == 0 and after_match_count <= 4:
|
||||
character_approved_count += 1
|
||||
continue
|
||||
|
||||
if (
|
||||
before_match_count / len(characters_before_source) >= 0.4
|
||||
or after_match_count / len(characters_after_source) >= 0.4
|
||||
):
|
||||
character_approved_count += 1
|
||||
continue
|
||||
|
||||
return character_approved_count / len(ordered_characters)
|
||||
|
||||
|
||||
def alpha_unicode_split(decoded_sequence: str) -> list[str]:
|
||||
"""
|
||||
Given a decoded text sequence, return a list of str. Unicode range / alphabet separation.
|
||||
Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list;
|
||||
One containing the latin letters and the other hebrew.
|
||||
"""
|
||||
layers: dict[str, str] = {}
|
||||
|
||||
for character in decoded_sequence:
|
||||
if character.isalpha() is False:
|
||||
continue
|
||||
|
||||
character_range: str | None = unicode_range(character)
|
||||
|
||||
if character_range is None:
|
||||
continue
|
||||
|
||||
layer_target_range: str | None = None
|
||||
|
||||
for discovered_range in layers:
|
||||
if (
|
||||
is_suspiciously_successive_range(discovered_range, character_range)
|
||||
is False
|
||||
):
|
||||
layer_target_range = discovered_range
|
||||
break
|
||||
|
||||
if layer_target_range is None:
|
||||
layer_target_range = character_range
|
||||
|
||||
if layer_target_range not in layers:
|
||||
layers[layer_target_range] = character.lower()
|
||||
continue
|
||||
|
||||
layers[layer_target_range] += character.lower()
|
||||
|
||||
return list(layers.values())
|
||||
|
||||
|
||||
def merge_coherence_ratios(results: list[CoherenceMatches]) -> CoherenceMatches:
|
||||
"""
|
||||
This function merge results previously given by the function coherence_ratio.
|
||||
The return type is the same as coherence_ratio.
|
||||
"""
|
||||
per_language_ratios: dict[str, list[float]] = {}
|
||||
for result in results:
|
||||
for sub_result in result:
|
||||
language, ratio = sub_result
|
||||
if language not in per_language_ratios:
|
||||
per_language_ratios[language] = [ratio]
|
||||
continue
|
||||
per_language_ratios[language].append(ratio)
|
||||
|
||||
merge = [
|
||||
(
|
||||
language,
|
||||
round(
|
||||
sum(per_language_ratios[language]) / len(per_language_ratios[language]),
|
||||
4,
|
||||
),
|
||||
)
|
||||
for language in per_language_ratios
|
||||
]
|
||||
|
||||
return sorted(merge, key=lambda x: x[1], reverse=True)
|
||||
|
||||
|
||||
def filter_alt_coherence_matches(results: CoherenceMatches) -> CoherenceMatches:
|
||||
"""
|
||||
We shall NOT return "English—" in CoherenceMatches because it is an alternative
|
||||
of "English". This function only keeps the best match and remove the em-dash in it.
|
||||
"""
|
||||
index_results: dict[str, list[float]] = dict()
|
||||
|
||||
for result in results:
|
||||
language, ratio = result
|
||||
no_em_name: str = language.replace("—", "")
|
||||
|
||||
if no_em_name not in index_results:
|
||||
index_results[no_em_name] = []
|
||||
|
||||
index_results[no_em_name].append(ratio)
|
||||
|
||||
if any(len(index_results[e]) > 1 for e in index_results):
|
||||
filtered_results: CoherenceMatches = []
|
||||
|
||||
for language in index_results:
|
||||
filtered_results.append((language, max(index_results[language])))
|
||||
|
||||
return filtered_results
|
||||
|
||||
return results
|
||||
|
||||
|
||||
@lru_cache(maxsize=2048)
|
||||
def coherence_ratio(
|
||||
decoded_sequence: str, threshold: float = 0.1, lg_inclusion: str | None = None
|
||||
) -> CoherenceMatches:
|
||||
"""
|
||||
Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers.
|
||||
A layer = Character extraction by alphabets/ranges.
|
||||
"""
|
||||
|
||||
results: list[tuple[str, float]] = []
|
||||
ignore_non_latin: bool = False
|
||||
|
||||
sufficient_match_count: int = 0
|
||||
|
||||
lg_inclusion_list = lg_inclusion.split(",") if lg_inclusion is not None else []
|
||||
if "Latin Based" in lg_inclusion_list:
|
||||
ignore_non_latin = True
|
||||
lg_inclusion_list.remove("Latin Based")
|
||||
|
||||
for layer in alpha_unicode_split(decoded_sequence):
|
||||
sequence_frequencies: TypeCounter[str] = Counter(layer)
|
||||
most_common = sequence_frequencies.most_common()
|
||||
|
||||
character_count: int = sum(o for c, o in most_common)
|
||||
|
||||
if character_count <= TOO_SMALL_SEQUENCE:
|
||||
continue
|
||||
|
||||
popular_character_ordered: list[str] = [c for c, o in most_common]
|
||||
|
||||
for language in lg_inclusion_list or alphabet_languages(
|
||||
popular_character_ordered, ignore_non_latin
|
||||
):
|
||||
ratio: float = characters_popularity_compare(
|
||||
language, popular_character_ordered
|
||||
)
|
||||
|
||||
if ratio < threshold:
|
||||
continue
|
||||
elif ratio >= 0.8:
|
||||
sufficient_match_count += 1
|
||||
|
||||
results.append((language, round(ratio, 4)))
|
||||
|
||||
if sufficient_match_count >= 3:
|
||||
break
|
||||
|
||||
return sorted(
|
||||
filter_alt_coherence_matches(results), key=lambda x: x[1], reverse=True
|
||||
)
|
||||
@@ -0,0 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .__main__ import cli_detect, query_yes_no
|
||||
|
||||
__all__ = (
|
||||
"cli_detect",
|
||||
"query_yes_no",
|
||||
)
|
||||
@@ -0,0 +1,381 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import typing
|
||||
from json import dumps
|
||||
from os.path import abspath, basename, dirname, join, realpath
|
||||
from platform import python_version
|
||||
from unicodedata import unidata_version
|
||||
|
||||
import charset_normalizer.md as md_module
|
||||
from charset_normalizer import from_fp
|
||||
from charset_normalizer.models import CliDetectionResult
|
||||
from charset_normalizer.version import __version__
|
||||
|
||||
|
||||
def query_yes_no(question: str, default: str = "yes") -> bool:
|
||||
"""Ask a yes/no question via input() and return their answer.
|
||||
|
||||
"question" is a string that is presented to the user.
|
||||
"default" is the presumed answer if the user just hits <Enter>.
|
||||
It must be "yes" (the default), "no" or None (meaning
|
||||
an answer is required of the user).
|
||||
|
||||
The "answer" return value is True for "yes" or False for "no".
|
||||
|
||||
Credit goes to (c) https://stackoverflow.com/questions/3041986/apt-command-line-interface-like-yes-no-input
|
||||
"""
|
||||
valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
|
||||
if default is None:
|
||||
prompt = " [y/n] "
|
||||
elif default == "yes":
|
||||
prompt = " [Y/n] "
|
||||
elif default == "no":
|
||||
prompt = " [y/N] "
|
||||
else:
|
||||
raise ValueError("invalid default answer: '%s'" % default)
|
||||
|
||||
while True:
|
||||
sys.stdout.write(question + prompt)
|
||||
choice = input().lower()
|
||||
if default is not None and choice == "":
|
||||
return valid[default]
|
||||
elif choice in valid:
|
||||
return valid[choice]
|
||||
else:
|
||||
sys.stdout.write("Please respond with 'yes' or 'no' (or 'y' or 'n').\n")
|
||||
|
||||
|
||||
class FileType:
|
||||
"""Factory for creating file object types
|
||||
|
||||
Instances of FileType are typically passed as type= arguments to the
|
||||
ArgumentParser add_argument() method.
|
||||
|
||||
Keyword Arguments:
|
||||
- mode -- A string indicating how the file is to be opened. Accepts the
|
||||
same values as the builtin open() function.
|
||||
- bufsize -- The file's desired buffer size. Accepts the same values as
|
||||
the builtin open() function.
|
||||
- encoding -- The file's encoding. Accepts the same values as the
|
||||
builtin open() function.
|
||||
- errors -- A string indicating how encoding and decoding errors are to
|
||||
be handled. Accepts the same value as the builtin open() function.
|
||||
|
||||
Backported from CPython 3.12
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mode: str = "r",
|
||||
bufsize: int = -1,
|
||||
encoding: str | None = None,
|
||||
errors: str | None = None,
|
||||
):
|
||||
self._mode = mode
|
||||
self._bufsize = bufsize
|
||||
self._encoding = encoding
|
||||
self._errors = errors
|
||||
|
||||
def __call__(self, string: str) -> typing.IO: # type: ignore[type-arg]
|
||||
# the special argument "-" means sys.std{in,out}
|
||||
if string == "-":
|
||||
if "r" in self._mode:
|
||||
return sys.stdin.buffer if "b" in self._mode else sys.stdin
|
||||
elif any(c in self._mode for c in "wax"):
|
||||
return sys.stdout.buffer if "b" in self._mode else sys.stdout
|
||||
else:
|
||||
msg = f'argument "-" with mode {self._mode}'
|
||||
raise ValueError(msg)
|
||||
|
||||
# all other arguments are used as file names
|
||||
try:
|
||||
return open(string, self._mode, self._bufsize, self._encoding, self._errors)
|
||||
except OSError as e:
|
||||
message = f"can't open '{string}': {e}"
|
||||
raise argparse.ArgumentTypeError(message)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
args = self._mode, self._bufsize
|
||||
kwargs = [("encoding", self._encoding), ("errors", self._errors)]
|
||||
args_str = ", ".join(
|
||||
[repr(arg) for arg in args if arg != -1]
|
||||
+ [f"{kw}={arg!r}" for kw, arg in kwargs if arg is not None]
|
||||
)
|
||||
return f"{type(self).__name__}({args_str})"
|
||||
|
||||
|
||||
def cli_detect(argv: list[str] | None = None) -> int:
|
||||
"""
|
||||
CLI assistant using ARGV and ArgumentParser
|
||||
:param argv:
|
||||
:return: 0 if everything is fine, anything else equal trouble
|
||||
"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="The Real First Universal Charset Detector. "
|
||||
"Discover originating encoding used on text file. "
|
||||
"Normalize text to unicode."
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"files", type=FileType("rb"), nargs="+", help="File(s) to be analysed"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="verbose",
|
||||
help="Display complementary information about file if any. "
|
||||
"Stdout will contain logs about the detection process.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-a",
|
||||
"--with-alternative",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="alternatives",
|
||||
help="Output complementary possibilities if any. Top-level JSON WILL be a list.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-n",
|
||||
"--normalize",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="normalize",
|
||||
help="Permit to normalize input file. If not set, program does not write anything.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-m",
|
||||
"--minimal",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="minimal",
|
||||
help="Only output the charset detected to STDOUT. Disabling JSON output.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-r",
|
||||
"--replace",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="replace",
|
||||
help="Replace file when trying to normalize it instead of creating a new one.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--force",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="force",
|
||||
help="Replace file without asking if you are sure, use this flag with caution.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-i",
|
||||
"--no-preemptive",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="no_preemptive",
|
||||
help="Disable looking at a charset declaration to hint the detector.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--threshold",
|
||||
action="store",
|
||||
default=0.2,
|
||||
type=float,
|
||||
dest="threshold",
|
||||
help="Define a custom maximum amount of noise allowed in decoded content. 0. <= noise <= 1.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
action="version",
|
||||
version="Charset-Normalizer {} - Python {} - Unicode {} - SpeedUp {}".format(
|
||||
__version__,
|
||||
python_version(),
|
||||
unidata_version,
|
||||
"OFF" if md_module.__file__.lower().endswith(".py") else "ON",
|
||||
),
|
||||
help="Show version information and exit.",
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.replace is True and args.normalize is False:
|
||||
if args.files:
|
||||
for my_file in args.files:
|
||||
my_file.close()
|
||||
print("Use --replace in addition of --normalize only.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if args.force is True and args.replace is False:
|
||||
if args.files:
|
||||
for my_file in args.files:
|
||||
my_file.close()
|
||||
print("Use --force in addition of --replace only.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if args.threshold < 0.0 or args.threshold > 1.0:
|
||||
if args.files:
|
||||
for my_file in args.files:
|
||||
my_file.close()
|
||||
print("--threshold VALUE should be between 0. AND 1.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
x_ = []
|
||||
|
||||
for my_file in args.files:
|
||||
matches = from_fp(
|
||||
my_file,
|
||||
threshold=args.threshold,
|
||||
explain=args.verbose,
|
||||
preemptive_behaviour=args.no_preemptive is False,
|
||||
)
|
||||
|
||||
best_guess = matches.best()
|
||||
|
||||
if best_guess is None:
|
||||
print(
|
||||
'Unable to identify originating encoding for "{}". {}'.format(
|
||||
my_file.name,
|
||||
(
|
||||
"Maybe try increasing maximum amount of chaos."
|
||||
if args.threshold < 1.0
|
||||
else ""
|
||||
),
|
||||
),
|
||||
file=sys.stderr,
|
||||
)
|
||||
x_.append(
|
||||
CliDetectionResult(
|
||||
abspath(my_file.name),
|
||||
None,
|
||||
[],
|
||||
[],
|
||||
"Unknown",
|
||||
[],
|
||||
False,
|
||||
1.0,
|
||||
0.0,
|
||||
None,
|
||||
True,
|
||||
)
|
||||
)
|
||||
else:
|
||||
x_.append(
|
||||
CliDetectionResult(
|
||||
abspath(my_file.name),
|
||||
best_guess.encoding,
|
||||
best_guess.encoding_aliases,
|
||||
[
|
||||
cp
|
||||
for cp in best_guess.could_be_from_charset
|
||||
if cp != best_guess.encoding
|
||||
],
|
||||
best_guess.language,
|
||||
best_guess.alphabets,
|
||||
best_guess.bom,
|
||||
best_guess.percent_chaos,
|
||||
best_guess.percent_coherence,
|
||||
None,
|
||||
True,
|
||||
)
|
||||
)
|
||||
|
||||
if len(matches) > 1 and args.alternatives:
|
||||
for el in matches:
|
||||
if el != best_guess:
|
||||
x_.append(
|
||||
CliDetectionResult(
|
||||
abspath(my_file.name),
|
||||
el.encoding,
|
||||
el.encoding_aliases,
|
||||
[
|
||||
cp
|
||||
for cp in el.could_be_from_charset
|
||||
if cp != el.encoding
|
||||
],
|
||||
el.language,
|
||||
el.alphabets,
|
||||
el.bom,
|
||||
el.percent_chaos,
|
||||
el.percent_coherence,
|
||||
None,
|
||||
False,
|
||||
)
|
||||
)
|
||||
|
||||
if args.normalize is True:
|
||||
if best_guess.encoding.startswith("utf") is True:
|
||||
print(
|
||||
'"{}" file does not need to be normalized, as it already came from unicode.'.format(
|
||||
my_file.name
|
||||
),
|
||||
file=sys.stderr,
|
||||
)
|
||||
if my_file.closed is False:
|
||||
my_file.close()
|
||||
continue
|
||||
|
||||
dir_path = dirname(realpath(my_file.name))
|
||||
file_name = basename(realpath(my_file.name))
|
||||
|
||||
o_: list[str] = file_name.split(".")
|
||||
|
||||
if args.replace is False:
|
||||
o_.insert(-1, best_guess.encoding)
|
||||
if my_file.closed is False:
|
||||
my_file.close()
|
||||
elif (
|
||||
args.force is False
|
||||
and query_yes_no(
|
||||
'Are you sure to normalize "{}" by replacing it ?'.format(
|
||||
my_file.name
|
||||
),
|
||||
"no",
|
||||
)
|
||||
is False
|
||||
):
|
||||
if my_file.closed is False:
|
||||
my_file.close()
|
||||
continue
|
||||
|
||||
try:
|
||||
x_[0].unicode_path = join(dir_path, ".".join(o_))
|
||||
|
||||
with open(x_[0].unicode_path, "wb") as fp:
|
||||
fp.write(best_guess.output())
|
||||
except OSError as e:
|
||||
print(str(e), file=sys.stderr)
|
||||
if my_file.closed is False:
|
||||
my_file.close()
|
||||
return 2
|
||||
|
||||
if my_file.closed is False:
|
||||
my_file.close()
|
||||
|
||||
if args.minimal is False:
|
||||
print(
|
||||
dumps(
|
||||
[el.__dict__ for el in x_] if len(x_) > 1 else x_[0].__dict__,
|
||||
ensure_ascii=True,
|
||||
indent=4,
|
||||
)
|
||||
)
|
||||
else:
|
||||
for my_file in args.files:
|
||||
print(
|
||||
", ".join(
|
||||
[
|
||||
el.encoding or "undefined"
|
||||
for el in x_
|
||||
if el.path == abspath(my_file.name)
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli_detect()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from warnings import warn
|
||||
|
||||
from .api import from_bytes
|
||||
from .constant import CHARDET_CORRESPONDENCE, TOO_SMALL_SEQUENCE
|
||||
|
||||
# TODO: remove this check when dropping Python 3.7 support
|
||||
if TYPE_CHECKING:
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
class ResultDict(TypedDict):
|
||||
encoding: str | None
|
||||
language: str
|
||||
confidence: float | None
|
||||
|
||||
|
||||
def detect(
|
||||
byte_str: bytes, should_rename_legacy: bool = False, **kwargs: Any
|
||||
) -> ResultDict:
|
||||
"""
|
||||
chardet legacy method
|
||||
Detect the encoding of the given byte string. It should be mostly backward-compatible.
|
||||
Encoding name will match Chardet own writing whenever possible. (Not on encoding name unsupported by it)
|
||||
This function is deprecated and should be used to migrate your project easily, consult the documentation for
|
||||
further information. Not planned for removal.
|
||||
|
||||
:param byte_str: The byte sequence to examine.
|
||||
:param should_rename_legacy: Should we rename legacy encodings
|
||||
to their more modern equivalents?
|
||||
"""
|
||||
if len(kwargs):
|
||||
warn(
|
||||
f"charset-normalizer disregard arguments '{','.join(list(kwargs.keys()))}' in legacy function detect()"
|
||||
)
|
||||
|
||||
if not isinstance(byte_str, (bytearray, bytes)):
|
||||
raise TypeError( # pragma: nocover
|
||||
f"Expected object of type bytes or bytearray, got: {type(byte_str)}"
|
||||
)
|
||||
|
||||
if isinstance(byte_str, bytearray):
|
||||
byte_str = bytes(byte_str)
|
||||
|
||||
r = from_bytes(byte_str).best()
|
||||
|
||||
encoding = r.encoding if r is not None else None
|
||||
language = r.language if r is not None and r.language != "Unknown" else ""
|
||||
confidence = 1.0 - r.chaos if r is not None else None
|
||||
|
||||
# automatically lower confidence
|
||||
# on small bytes samples.
|
||||
# https://github.com/jawah/charset_normalizer/issues/391
|
||||
if (
|
||||
confidence is not None
|
||||
and confidence >= 0.9
|
||||
and encoding
|
||||
not in {
|
||||
"utf_8",
|
||||
"ascii",
|
||||
}
|
||||
and r.bom is False # type: ignore[union-attr]
|
||||
and len(byte_str) < TOO_SMALL_SEQUENCE
|
||||
):
|
||||
confidence -= 0.2
|
||||
|
||||
# Note: CharsetNormalizer does not return 'UTF-8-SIG' as the sig get stripped in the detection/normalization process
|
||||
# but chardet does return 'utf-8-sig' and it is a valid codec name.
|
||||
if r is not None and encoding == "utf_8" and r.bom:
|
||||
encoding += "_sig"
|
||||
|
||||
if should_rename_legacy is False and encoding in CHARDET_CORRESPONDENCE:
|
||||
encoding = CHARDET_CORRESPONDENCE[encoding]
|
||||
|
||||
return {
|
||||
"encoding": encoding,
|
||||
"language": language,
|
||||
"confidence": confidence,
|
||||
}
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,635 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from logging import getLogger
|
||||
|
||||
from .constant import (
|
||||
COMMON_SAFE_ASCII_CHARACTERS,
|
||||
TRACE,
|
||||
UNICODE_SECONDARY_RANGE_KEYWORD,
|
||||
)
|
||||
from .utils import (
|
||||
is_accentuated,
|
||||
is_arabic,
|
||||
is_arabic_isolated_form,
|
||||
is_case_variable,
|
||||
is_cjk,
|
||||
is_emoticon,
|
||||
is_hangul,
|
||||
is_hiragana,
|
||||
is_katakana,
|
||||
is_latin,
|
||||
is_punctuation,
|
||||
is_separator,
|
||||
is_symbol,
|
||||
is_thai,
|
||||
is_unprintable,
|
||||
remove_accent,
|
||||
unicode_range,
|
||||
is_cjk_uncommon,
|
||||
)
|
||||
|
||||
|
||||
class MessDetectorPlugin:
|
||||
"""
|
||||
Base abstract class used for mess detection plugins.
|
||||
All detectors MUST extend and implement given methods.
|
||||
"""
|
||||
|
||||
def eligible(self, character: str) -> bool:
|
||||
"""
|
||||
Determine if given character should be fed in.
|
||||
"""
|
||||
raise NotImplementedError # pragma: nocover
|
||||
|
||||
def feed(self, character: str) -> None:
|
||||
"""
|
||||
The main routine to be executed upon character.
|
||||
Insert the logic in witch the text would be considered chaotic.
|
||||
"""
|
||||
raise NotImplementedError # pragma: nocover
|
||||
|
||||
def reset(self) -> None: # pragma: no cover
|
||||
"""
|
||||
Permit to reset the plugin to the initial state.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def ratio(self) -> float:
|
||||
"""
|
||||
Compute the chaos ratio based on what your feed() has seen.
|
||||
Must NOT be lower than 0.; No restriction gt 0.
|
||||
"""
|
||||
raise NotImplementedError # pragma: nocover
|
||||
|
||||
|
||||
class TooManySymbolOrPunctuationPlugin(MessDetectorPlugin):
|
||||
def __init__(self) -> None:
|
||||
self._punctuation_count: int = 0
|
||||
self._symbol_count: int = 0
|
||||
self._character_count: int = 0
|
||||
|
||||
self._last_printable_char: str | None = None
|
||||
self._frenzy_symbol_in_word: bool = False
|
||||
|
||||
def eligible(self, character: str) -> bool:
|
||||
return character.isprintable()
|
||||
|
||||
def feed(self, character: str) -> None:
|
||||
self._character_count += 1
|
||||
|
||||
if (
|
||||
character != self._last_printable_char
|
||||
and character not in COMMON_SAFE_ASCII_CHARACTERS
|
||||
):
|
||||
if is_punctuation(character):
|
||||
self._punctuation_count += 1
|
||||
elif (
|
||||
character.isdigit() is False
|
||||
and is_symbol(character)
|
||||
and is_emoticon(character) is False
|
||||
):
|
||||
self._symbol_count += 2
|
||||
|
||||
self._last_printable_char = character
|
||||
|
||||
def reset(self) -> None: # Abstract
|
||||
self._punctuation_count = 0
|
||||
self._character_count = 0
|
||||
self._symbol_count = 0
|
||||
|
||||
@property
|
||||
def ratio(self) -> float:
|
||||
if self._character_count == 0:
|
||||
return 0.0
|
||||
|
||||
ratio_of_punctuation: float = (
|
||||
self._punctuation_count + self._symbol_count
|
||||
) / self._character_count
|
||||
|
||||
return ratio_of_punctuation if ratio_of_punctuation >= 0.3 else 0.0
|
||||
|
||||
|
||||
class TooManyAccentuatedPlugin(MessDetectorPlugin):
|
||||
def __init__(self) -> None:
|
||||
self._character_count: int = 0
|
||||
self._accentuated_count: int = 0
|
||||
|
||||
def eligible(self, character: str) -> bool:
|
||||
return character.isalpha()
|
||||
|
||||
def feed(self, character: str) -> None:
|
||||
self._character_count += 1
|
||||
|
||||
if is_accentuated(character):
|
||||
self._accentuated_count += 1
|
||||
|
||||
def reset(self) -> None: # Abstract
|
||||
self._character_count = 0
|
||||
self._accentuated_count = 0
|
||||
|
||||
@property
|
||||
def ratio(self) -> float:
|
||||
if self._character_count < 8:
|
||||
return 0.0
|
||||
|
||||
ratio_of_accentuation: float = self._accentuated_count / self._character_count
|
||||
return ratio_of_accentuation if ratio_of_accentuation >= 0.35 else 0.0
|
||||
|
||||
|
||||
class UnprintablePlugin(MessDetectorPlugin):
|
||||
def __init__(self) -> None:
|
||||
self._unprintable_count: int = 0
|
||||
self._character_count: int = 0
|
||||
|
||||
def eligible(self, character: str) -> bool:
|
||||
return True
|
||||
|
||||
def feed(self, character: str) -> None:
|
||||
if is_unprintable(character):
|
||||
self._unprintable_count += 1
|
||||
self._character_count += 1
|
||||
|
||||
def reset(self) -> None: # Abstract
|
||||
self._unprintable_count = 0
|
||||
|
||||
@property
|
||||
def ratio(self) -> float:
|
||||
if self._character_count == 0:
|
||||
return 0.0
|
||||
|
||||
return (self._unprintable_count * 8) / self._character_count
|
||||
|
||||
|
||||
class SuspiciousDuplicateAccentPlugin(MessDetectorPlugin):
|
||||
def __init__(self) -> None:
|
||||
self._successive_count: int = 0
|
||||
self._character_count: int = 0
|
||||
|
||||
self._last_latin_character: str | None = None
|
||||
|
||||
def eligible(self, character: str) -> bool:
|
||||
return character.isalpha() and is_latin(character)
|
||||
|
||||
def feed(self, character: str) -> None:
|
||||
self._character_count += 1
|
||||
if (
|
||||
self._last_latin_character is not None
|
||||
and is_accentuated(character)
|
||||
and is_accentuated(self._last_latin_character)
|
||||
):
|
||||
if character.isupper() and self._last_latin_character.isupper():
|
||||
self._successive_count += 1
|
||||
# Worse if its the same char duplicated with different accent.
|
||||
if remove_accent(character) == remove_accent(self._last_latin_character):
|
||||
self._successive_count += 1
|
||||
self._last_latin_character = character
|
||||
|
||||
def reset(self) -> None: # Abstract
|
||||
self._successive_count = 0
|
||||
self._character_count = 0
|
||||
self._last_latin_character = None
|
||||
|
||||
@property
|
||||
def ratio(self) -> float:
|
||||
if self._character_count == 0:
|
||||
return 0.0
|
||||
|
||||
return (self._successive_count * 2) / self._character_count
|
||||
|
||||
|
||||
class SuspiciousRange(MessDetectorPlugin):
|
||||
def __init__(self) -> None:
|
||||
self._suspicious_successive_range_count: int = 0
|
||||
self._character_count: int = 0
|
||||
self._last_printable_seen: str | None = None
|
||||
|
||||
def eligible(self, character: str) -> bool:
|
||||
return character.isprintable()
|
||||
|
||||
def feed(self, character: str) -> None:
|
||||
self._character_count += 1
|
||||
|
||||
if (
|
||||
character.isspace()
|
||||
or is_punctuation(character)
|
||||
or character in COMMON_SAFE_ASCII_CHARACTERS
|
||||
):
|
||||
self._last_printable_seen = None
|
||||
return
|
||||
|
||||
if self._last_printable_seen is None:
|
||||
self._last_printable_seen = character
|
||||
return
|
||||
|
||||
unicode_range_a: str | None = unicode_range(self._last_printable_seen)
|
||||
unicode_range_b: str | None = unicode_range(character)
|
||||
|
||||
if is_suspiciously_successive_range(unicode_range_a, unicode_range_b):
|
||||
self._suspicious_successive_range_count += 1
|
||||
|
||||
self._last_printable_seen = character
|
||||
|
||||
def reset(self) -> None: # Abstract
|
||||
self._character_count = 0
|
||||
self._suspicious_successive_range_count = 0
|
||||
self._last_printable_seen = None
|
||||
|
||||
@property
|
||||
def ratio(self) -> float:
|
||||
if self._character_count <= 13:
|
||||
return 0.0
|
||||
|
||||
ratio_of_suspicious_range_usage: float = (
|
||||
self._suspicious_successive_range_count * 2
|
||||
) / self._character_count
|
||||
|
||||
return ratio_of_suspicious_range_usage
|
||||
|
||||
|
||||
class SuperWeirdWordPlugin(MessDetectorPlugin):
|
||||
def __init__(self) -> None:
|
||||
self._word_count: int = 0
|
||||
self._bad_word_count: int = 0
|
||||
self._foreign_long_count: int = 0
|
||||
|
||||
self._is_current_word_bad: bool = False
|
||||
self._foreign_long_watch: bool = False
|
||||
|
||||
self._character_count: int = 0
|
||||
self._bad_character_count: int = 0
|
||||
|
||||
self._buffer: str = ""
|
||||
self._buffer_accent_count: int = 0
|
||||
self._buffer_glyph_count: int = 0
|
||||
|
||||
def eligible(self, character: str) -> bool:
|
||||
return True
|
||||
|
||||
def feed(self, character: str) -> None:
|
||||
if character.isalpha():
|
||||
self._buffer += character
|
||||
if is_accentuated(character):
|
||||
self._buffer_accent_count += 1
|
||||
if (
|
||||
self._foreign_long_watch is False
|
||||
and (is_latin(character) is False or is_accentuated(character))
|
||||
and is_cjk(character) is False
|
||||
and is_hangul(character) is False
|
||||
and is_katakana(character) is False
|
||||
and is_hiragana(character) is False
|
||||
and is_thai(character) is False
|
||||
):
|
||||
self._foreign_long_watch = True
|
||||
if (
|
||||
is_cjk(character)
|
||||
or is_hangul(character)
|
||||
or is_katakana(character)
|
||||
or is_hiragana(character)
|
||||
or is_thai(character)
|
||||
):
|
||||
self._buffer_glyph_count += 1
|
||||
return
|
||||
if not self._buffer:
|
||||
return
|
||||
if (
|
||||
character.isspace() or is_punctuation(character) or is_separator(character)
|
||||
) and self._buffer:
|
||||
self._word_count += 1
|
||||
buffer_length: int = len(self._buffer)
|
||||
|
||||
self._character_count += buffer_length
|
||||
|
||||
if buffer_length >= 4:
|
||||
if self._buffer_accent_count / buffer_length >= 0.5:
|
||||
self._is_current_word_bad = True
|
||||
# Word/Buffer ending with an upper case accentuated letter are so rare,
|
||||
# that we will consider them all as suspicious. Same weight as foreign_long suspicious.
|
||||
elif (
|
||||
is_accentuated(self._buffer[-1])
|
||||
and self._buffer[-1].isupper()
|
||||
and all(_.isupper() for _ in self._buffer) is False
|
||||
):
|
||||
self._foreign_long_count += 1
|
||||
self._is_current_word_bad = True
|
||||
elif self._buffer_glyph_count == 1:
|
||||
self._is_current_word_bad = True
|
||||
self._foreign_long_count += 1
|
||||
if buffer_length >= 24 and self._foreign_long_watch:
|
||||
camel_case_dst = [
|
||||
i
|
||||
for c, i in zip(self._buffer, range(0, buffer_length))
|
||||
if c.isupper()
|
||||
]
|
||||
probable_camel_cased: bool = False
|
||||
|
||||
if camel_case_dst and (len(camel_case_dst) / buffer_length <= 0.3):
|
||||
probable_camel_cased = True
|
||||
|
||||
if not probable_camel_cased:
|
||||
self._foreign_long_count += 1
|
||||
self._is_current_word_bad = True
|
||||
|
||||
if self._is_current_word_bad:
|
||||
self._bad_word_count += 1
|
||||
self._bad_character_count += len(self._buffer)
|
||||
self._is_current_word_bad = False
|
||||
|
||||
self._foreign_long_watch = False
|
||||
self._buffer = ""
|
||||
self._buffer_accent_count = 0
|
||||
self._buffer_glyph_count = 0
|
||||
elif (
|
||||
character not in {"<", ">", "-", "=", "~", "|", "_"}
|
||||
and character.isdigit() is False
|
||||
and is_symbol(character)
|
||||
):
|
||||
self._is_current_word_bad = True
|
||||
self._buffer += character
|
||||
|
||||
def reset(self) -> None: # Abstract
|
||||
self._buffer = ""
|
||||
self._is_current_word_bad = False
|
||||
self._foreign_long_watch = False
|
||||
self._bad_word_count = 0
|
||||
self._word_count = 0
|
||||
self._character_count = 0
|
||||
self._bad_character_count = 0
|
||||
self._foreign_long_count = 0
|
||||
|
||||
@property
|
||||
def ratio(self) -> float:
|
||||
if self._word_count <= 10 and self._foreign_long_count == 0:
|
||||
return 0.0
|
||||
|
||||
return self._bad_character_count / self._character_count
|
||||
|
||||
|
||||
class CjkUncommonPlugin(MessDetectorPlugin):
|
||||
"""
|
||||
Detect messy CJK text that probably means nothing.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._character_count: int = 0
|
||||
self._uncommon_count: int = 0
|
||||
|
||||
def eligible(self, character: str) -> bool:
|
||||
return is_cjk(character)
|
||||
|
||||
def feed(self, character: str) -> None:
|
||||
self._character_count += 1
|
||||
|
||||
if is_cjk_uncommon(character):
|
||||
self._uncommon_count += 1
|
||||
return
|
||||
|
||||
def reset(self) -> None: # Abstract
|
||||
self._character_count = 0
|
||||
self._uncommon_count = 0
|
||||
|
||||
@property
|
||||
def ratio(self) -> float:
|
||||
if self._character_count < 8:
|
||||
return 0.0
|
||||
|
||||
uncommon_form_usage: float = self._uncommon_count / self._character_count
|
||||
|
||||
# we can be pretty sure it's garbage when uncommon characters are widely
|
||||
# used. otherwise it could just be traditional chinese for example.
|
||||
return uncommon_form_usage / 10 if uncommon_form_usage > 0.5 else 0.0
|
||||
|
||||
|
||||
class ArchaicUpperLowerPlugin(MessDetectorPlugin):
|
||||
def __init__(self) -> None:
|
||||
self._buf: bool = False
|
||||
|
||||
self._character_count_since_last_sep: int = 0
|
||||
|
||||
self._successive_upper_lower_count: int = 0
|
||||
self._successive_upper_lower_count_final: int = 0
|
||||
|
||||
self._character_count: int = 0
|
||||
|
||||
self._last_alpha_seen: str | None = None
|
||||
self._current_ascii_only: bool = True
|
||||
|
||||
def eligible(self, character: str) -> bool:
|
||||
return True
|
||||
|
||||
def feed(self, character: str) -> None:
|
||||
is_concerned = character.isalpha() and is_case_variable(character)
|
||||
chunk_sep = is_concerned is False
|
||||
|
||||
if chunk_sep and self._character_count_since_last_sep > 0:
|
||||
if (
|
||||
self._character_count_since_last_sep <= 64
|
||||
and character.isdigit() is False
|
||||
and self._current_ascii_only is False
|
||||
):
|
||||
self._successive_upper_lower_count_final += (
|
||||
self._successive_upper_lower_count
|
||||
)
|
||||
|
||||
self._successive_upper_lower_count = 0
|
||||
self._character_count_since_last_sep = 0
|
||||
self._last_alpha_seen = None
|
||||
self._buf = False
|
||||
self._character_count += 1
|
||||
self._current_ascii_only = True
|
||||
|
||||
return
|
||||
|
||||
if self._current_ascii_only is True and character.isascii() is False:
|
||||
self._current_ascii_only = False
|
||||
|
||||
if self._last_alpha_seen is not None:
|
||||
if (character.isupper() and self._last_alpha_seen.islower()) or (
|
||||
character.islower() and self._last_alpha_seen.isupper()
|
||||
):
|
||||
if self._buf is True:
|
||||
self._successive_upper_lower_count += 2
|
||||
self._buf = False
|
||||
else:
|
||||
self._buf = True
|
||||
else:
|
||||
self._buf = False
|
||||
|
||||
self._character_count += 1
|
||||
self._character_count_since_last_sep += 1
|
||||
self._last_alpha_seen = character
|
||||
|
||||
def reset(self) -> None: # Abstract
|
||||
self._character_count = 0
|
||||
self._character_count_since_last_sep = 0
|
||||
self._successive_upper_lower_count = 0
|
||||
self._successive_upper_lower_count_final = 0
|
||||
self._last_alpha_seen = None
|
||||
self._buf = False
|
||||
self._current_ascii_only = True
|
||||
|
||||
@property
|
||||
def ratio(self) -> float:
|
||||
if self._character_count == 0:
|
||||
return 0.0
|
||||
|
||||
return self._successive_upper_lower_count_final / self._character_count
|
||||
|
||||
|
||||
class ArabicIsolatedFormPlugin(MessDetectorPlugin):
|
||||
def __init__(self) -> None:
|
||||
self._character_count: int = 0
|
||||
self._isolated_form_count: int = 0
|
||||
|
||||
def reset(self) -> None: # Abstract
|
||||
self._character_count = 0
|
||||
self._isolated_form_count = 0
|
||||
|
||||
def eligible(self, character: str) -> bool:
|
||||
return is_arabic(character)
|
||||
|
||||
def feed(self, character: str) -> None:
|
||||
self._character_count += 1
|
||||
|
||||
if is_arabic_isolated_form(character):
|
||||
self._isolated_form_count += 1
|
||||
|
||||
@property
|
||||
def ratio(self) -> float:
|
||||
if self._character_count < 8:
|
||||
return 0.0
|
||||
|
||||
isolated_form_usage: float = self._isolated_form_count / self._character_count
|
||||
|
||||
return isolated_form_usage
|
||||
|
||||
|
||||
@lru_cache(maxsize=1024)
|
||||
def is_suspiciously_successive_range(
|
||||
unicode_range_a: str | None, unicode_range_b: str | None
|
||||
) -> bool:
|
||||
"""
|
||||
Determine if two Unicode range seen next to each other can be considered as suspicious.
|
||||
"""
|
||||
if unicode_range_a is None or unicode_range_b is None:
|
||||
return True
|
||||
|
||||
if unicode_range_a == unicode_range_b:
|
||||
return False
|
||||
|
||||
if "Latin" in unicode_range_a and "Latin" in unicode_range_b:
|
||||
return False
|
||||
|
||||
if "Emoticons" in unicode_range_a or "Emoticons" in unicode_range_b:
|
||||
return False
|
||||
|
||||
# Latin characters can be accompanied with a combining diacritical mark
|
||||
# eg. Vietnamese.
|
||||
if ("Latin" in unicode_range_a or "Latin" in unicode_range_b) and (
|
||||
"Combining" in unicode_range_a or "Combining" in unicode_range_b
|
||||
):
|
||||
return False
|
||||
|
||||
keywords_range_a, keywords_range_b = (
|
||||
unicode_range_a.split(" "),
|
||||
unicode_range_b.split(" "),
|
||||
)
|
||||
|
||||
for el in keywords_range_a:
|
||||
if el in UNICODE_SECONDARY_RANGE_KEYWORD:
|
||||
continue
|
||||
if el in keywords_range_b:
|
||||
return False
|
||||
|
||||
# Japanese Exception
|
||||
range_a_jp_chars, range_b_jp_chars = (
|
||||
unicode_range_a
|
||||
in (
|
||||
"Hiragana",
|
||||
"Katakana",
|
||||
),
|
||||
unicode_range_b in ("Hiragana", "Katakana"),
|
||||
)
|
||||
if (range_a_jp_chars or range_b_jp_chars) and (
|
||||
"CJK" in unicode_range_a or "CJK" in unicode_range_b
|
||||
):
|
||||
return False
|
||||
if range_a_jp_chars and range_b_jp_chars:
|
||||
return False
|
||||
|
||||
if "Hangul" in unicode_range_a or "Hangul" in unicode_range_b:
|
||||
if "CJK" in unicode_range_a or "CJK" in unicode_range_b:
|
||||
return False
|
||||
if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin":
|
||||
return False
|
||||
|
||||
# Chinese/Japanese use dedicated range for punctuation and/or separators.
|
||||
if ("CJK" in unicode_range_a or "CJK" in unicode_range_b) or (
|
||||
unicode_range_a in ["Katakana", "Hiragana"]
|
||||
and unicode_range_b in ["Katakana", "Hiragana"]
|
||||
):
|
||||
if "Punctuation" in unicode_range_a or "Punctuation" in unicode_range_b:
|
||||
return False
|
||||
if "Forms" in unicode_range_a or "Forms" in unicode_range_b:
|
||||
return False
|
||||
if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin":
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
@lru_cache(maxsize=2048)
|
||||
def mess_ratio(
|
||||
decoded_sequence: str, maximum_threshold: float = 0.2, debug: bool = False
|
||||
) -> float:
|
||||
"""
|
||||
Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
|
||||
"""
|
||||
|
||||
detectors: list[MessDetectorPlugin] = [
|
||||
md_class() for md_class in MessDetectorPlugin.__subclasses__()
|
||||
]
|
||||
|
||||
length: int = len(decoded_sequence) + 1
|
||||
|
||||
mean_mess_ratio: float = 0.0
|
||||
|
||||
if length < 512:
|
||||
intermediary_mean_mess_ratio_calc: int = 32
|
||||
elif length <= 1024:
|
||||
intermediary_mean_mess_ratio_calc = 64
|
||||
else:
|
||||
intermediary_mean_mess_ratio_calc = 128
|
||||
|
||||
for character, index in zip(decoded_sequence + "\n", range(length)):
|
||||
for detector in detectors:
|
||||
if detector.eligible(character):
|
||||
detector.feed(character)
|
||||
|
||||
if (
|
||||
index > 0 and index % intermediary_mean_mess_ratio_calc == 0
|
||||
) or index == length - 1:
|
||||
mean_mess_ratio = sum(dt.ratio for dt in detectors)
|
||||
|
||||
if mean_mess_ratio >= maximum_threshold:
|
||||
break
|
||||
|
||||
if debug:
|
||||
logger = getLogger("charset_normalizer")
|
||||
|
||||
logger.log(
|
||||
TRACE,
|
||||
"Mess-detector extended-analysis start. "
|
||||
f"intermediary_mean_mess_ratio_calc={intermediary_mean_mess_ratio_calc} mean_mess_ratio={mean_mess_ratio} "
|
||||
f"maximum_threshold={maximum_threshold}",
|
||||
)
|
||||
|
||||
if len(decoded_sequence) > 16:
|
||||
logger.log(TRACE, f"Starting with: {decoded_sequence[:16]}")
|
||||
logger.log(TRACE, f"Ending with: {decoded_sequence[-16::]}")
|
||||
|
||||
for dt in detectors:
|
||||
logger.log(TRACE, f"{dt.__class__}: {dt.ratio}")
|
||||
|
||||
return round(mean_mess_ratio, 3)
|
||||
Executable
BIN
Binary file not shown.
@@ -0,0 +1,360 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from encodings.aliases import aliases
|
||||
from hashlib import sha256
|
||||
from json import dumps
|
||||
from re import sub
|
||||
from typing import Any, Iterator, List, Tuple
|
||||
|
||||
from .constant import RE_POSSIBLE_ENCODING_INDICATION, TOO_BIG_SEQUENCE
|
||||
from .utils import iana_name, is_multi_byte_encoding, unicode_range
|
||||
|
||||
|
||||
class CharsetMatch:
|
||||
def __init__(
|
||||
self,
|
||||
payload: bytes,
|
||||
guessed_encoding: str,
|
||||
mean_mess_ratio: float,
|
||||
has_sig_or_bom: bool,
|
||||
languages: CoherenceMatches,
|
||||
decoded_payload: str | None = None,
|
||||
preemptive_declaration: str | None = None,
|
||||
):
|
||||
self._payload: bytes = payload
|
||||
|
||||
self._encoding: str = guessed_encoding
|
||||
self._mean_mess_ratio: float = mean_mess_ratio
|
||||
self._languages: CoherenceMatches = languages
|
||||
self._has_sig_or_bom: bool = has_sig_or_bom
|
||||
self._unicode_ranges: list[str] | None = None
|
||||
|
||||
self._leaves: list[CharsetMatch] = []
|
||||
self._mean_coherence_ratio: float = 0.0
|
||||
|
||||
self._output_payload: bytes | None = None
|
||||
self._output_encoding: str | None = None
|
||||
|
||||
self._string: str | None = decoded_payload
|
||||
|
||||
self._preemptive_declaration: str | None = preemptive_declaration
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if not isinstance(other, CharsetMatch):
|
||||
if isinstance(other, str):
|
||||
return iana_name(other) == self.encoding
|
||||
return False
|
||||
return self.encoding == other.encoding and self.fingerprint == other.fingerprint
|
||||
|
||||
def __lt__(self, other: object) -> bool:
|
||||
"""
|
||||
Implemented to make sorted available upon CharsetMatches items.
|
||||
"""
|
||||
if not isinstance(other, CharsetMatch):
|
||||
raise ValueError
|
||||
|
||||
chaos_difference: float = abs(self.chaos - other.chaos)
|
||||
coherence_difference: float = abs(self.coherence - other.coherence)
|
||||
|
||||
# Below 1% difference --> Use Coherence
|
||||
if chaos_difference < 0.01 and coherence_difference > 0.02:
|
||||
return self.coherence > other.coherence
|
||||
elif chaos_difference < 0.01 and coherence_difference <= 0.02:
|
||||
# When having a difficult decision, use the result that decoded as many multi-byte as possible.
|
||||
# preserve RAM usage!
|
||||
if len(self._payload) >= TOO_BIG_SEQUENCE:
|
||||
return self.chaos < other.chaos
|
||||
return self.multi_byte_usage > other.multi_byte_usage
|
||||
|
||||
return self.chaos < other.chaos
|
||||
|
||||
@property
|
||||
def multi_byte_usage(self) -> float:
|
||||
return 1.0 - (len(str(self)) / len(self.raw))
|
||||
|
||||
def __str__(self) -> str:
|
||||
# Lazy Str Loading
|
||||
if self._string is None:
|
||||
self._string = str(self._payload, self._encoding, "strict")
|
||||
return self._string
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<CharsetMatch '{self.encoding}' bytes({self.fingerprint})>"
|
||||
|
||||
def add_submatch(self, other: CharsetMatch) -> None:
|
||||
if not isinstance(other, CharsetMatch) or other == self:
|
||||
raise ValueError(
|
||||
"Unable to add instance <{}> as a submatch of a CharsetMatch".format(
|
||||
other.__class__
|
||||
)
|
||||
)
|
||||
|
||||
other._string = None # Unload RAM usage; dirty trick.
|
||||
self._leaves.append(other)
|
||||
|
||||
@property
|
||||
def encoding(self) -> str:
|
||||
return self._encoding
|
||||
|
||||
@property
|
||||
def encoding_aliases(self) -> list[str]:
|
||||
"""
|
||||
Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855.
|
||||
"""
|
||||
also_known_as: list[str] = []
|
||||
for u, p in aliases.items():
|
||||
if self.encoding == u:
|
||||
also_known_as.append(p)
|
||||
elif self.encoding == p:
|
||||
also_known_as.append(u)
|
||||
return also_known_as
|
||||
|
||||
@property
|
||||
def bom(self) -> bool:
|
||||
return self._has_sig_or_bom
|
||||
|
||||
@property
|
||||
def byte_order_mark(self) -> bool:
|
||||
return self._has_sig_or_bom
|
||||
|
||||
@property
|
||||
def languages(self) -> list[str]:
|
||||
"""
|
||||
Return the complete list of possible languages found in decoded sequence.
|
||||
Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'.
|
||||
"""
|
||||
return [e[0] for e in self._languages]
|
||||
|
||||
@property
|
||||
def language(self) -> str:
|
||||
"""
|
||||
Most probable language found in decoded sequence. If none were detected or inferred, the property will return
|
||||
"Unknown".
|
||||
"""
|
||||
if not self._languages:
|
||||
# Trying to infer the language based on the given encoding
|
||||
# Its either English or we should not pronounce ourselves in certain cases.
|
||||
if "ascii" in self.could_be_from_charset:
|
||||
return "English"
|
||||
|
||||
# doing it there to avoid circular import
|
||||
from charset_normalizer.cd import encoding_languages, mb_encoding_languages
|
||||
|
||||
languages = (
|
||||
mb_encoding_languages(self.encoding)
|
||||
if is_multi_byte_encoding(self.encoding)
|
||||
else encoding_languages(self.encoding)
|
||||
)
|
||||
|
||||
if len(languages) == 0 or "Latin Based" in languages:
|
||||
return "Unknown"
|
||||
|
||||
return languages[0]
|
||||
|
||||
return self._languages[0][0]
|
||||
|
||||
@property
|
||||
def chaos(self) -> float:
|
||||
return self._mean_mess_ratio
|
||||
|
||||
@property
|
||||
def coherence(self) -> float:
|
||||
if not self._languages:
|
||||
return 0.0
|
||||
return self._languages[0][1]
|
||||
|
||||
@property
|
||||
def percent_chaos(self) -> float:
|
||||
return round(self.chaos * 100, ndigits=3)
|
||||
|
||||
@property
|
||||
def percent_coherence(self) -> float:
|
||||
return round(self.coherence * 100, ndigits=3)
|
||||
|
||||
@property
|
||||
def raw(self) -> bytes:
|
||||
"""
|
||||
Original untouched bytes.
|
||||
"""
|
||||
return self._payload
|
||||
|
||||
@property
|
||||
def submatch(self) -> list[CharsetMatch]:
|
||||
return self._leaves
|
||||
|
||||
@property
|
||||
def has_submatch(self) -> bool:
|
||||
return len(self._leaves) > 0
|
||||
|
||||
@property
|
||||
def alphabets(self) -> list[str]:
|
||||
if self._unicode_ranges is not None:
|
||||
return self._unicode_ranges
|
||||
# list detected ranges
|
||||
detected_ranges: list[str | None] = [unicode_range(char) for char in str(self)]
|
||||
# filter and sort
|
||||
self._unicode_ranges = sorted(list({r for r in detected_ranges if r}))
|
||||
return self._unicode_ranges
|
||||
|
||||
@property
|
||||
def could_be_from_charset(self) -> list[str]:
|
||||
"""
|
||||
The complete list of encoding that output the exact SAME str result and therefore could be the originating
|
||||
encoding.
|
||||
This list does include the encoding available in property 'encoding'.
|
||||
"""
|
||||
return [self._encoding] + [m.encoding for m in self._leaves]
|
||||
|
||||
def output(self, encoding: str = "utf_8") -> bytes:
|
||||
"""
|
||||
Method to get re-encoded bytes payload using given target encoding. Default to UTF-8.
|
||||
Any errors will be simply ignored by the encoder NOT replaced.
|
||||
"""
|
||||
if self._output_encoding is None or self._output_encoding != encoding:
|
||||
self._output_encoding = encoding
|
||||
decoded_string = str(self)
|
||||
if (
|
||||
self._preemptive_declaration is not None
|
||||
and self._preemptive_declaration.lower()
|
||||
not in ["utf-8", "utf8", "utf_8"]
|
||||
):
|
||||
patched_header = sub(
|
||||
RE_POSSIBLE_ENCODING_INDICATION,
|
||||
lambda m: m.string[m.span()[0] : m.span()[1]].replace(
|
||||
m.groups()[0],
|
||||
iana_name(self._output_encoding).replace("_", "-"), # type: ignore[arg-type]
|
||||
),
|
||||
decoded_string[:8192],
|
||||
count=1,
|
||||
)
|
||||
|
||||
decoded_string = patched_header + decoded_string[8192:]
|
||||
|
||||
self._output_payload = decoded_string.encode(encoding, "replace")
|
||||
|
||||
return self._output_payload # type: ignore
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> str:
|
||||
"""
|
||||
Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one.
|
||||
"""
|
||||
return sha256(self.output()).hexdigest()
|
||||
|
||||
|
||||
class CharsetMatches:
|
||||
"""
|
||||
Container with every CharsetMatch items ordered by default from most probable to the less one.
|
||||
Act like a list(iterable) but does not implements all related methods.
|
||||
"""
|
||||
|
||||
def __init__(self, results: list[CharsetMatch] | None = None):
|
||||
self._results: list[CharsetMatch] = sorted(results) if results else []
|
||||
|
||||
def __iter__(self) -> Iterator[CharsetMatch]:
|
||||
yield from self._results
|
||||
|
||||
def __getitem__(self, item: int | str) -> CharsetMatch:
|
||||
"""
|
||||
Retrieve a single item either by its position or encoding name (alias may be used here).
|
||||
Raise KeyError upon invalid index or encoding not present in results.
|
||||
"""
|
||||
if isinstance(item, int):
|
||||
return self._results[item]
|
||||
if isinstance(item, str):
|
||||
item = iana_name(item, False)
|
||||
for result in self._results:
|
||||
if item in result.could_be_from_charset:
|
||||
return result
|
||||
raise KeyError
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._results)
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
return len(self._results) > 0
|
||||
|
||||
def append(self, item: CharsetMatch) -> None:
|
||||
"""
|
||||
Insert a single match. Will be inserted accordingly to preserve sort.
|
||||
Can be inserted as a submatch.
|
||||
"""
|
||||
if not isinstance(item, CharsetMatch):
|
||||
raise ValueError(
|
||||
"Cannot append instance '{}' to CharsetMatches".format(
|
||||
str(item.__class__)
|
||||
)
|
||||
)
|
||||
# We should disable the submatch factoring when the input file is too heavy (conserve RAM usage)
|
||||
if len(item.raw) < TOO_BIG_SEQUENCE:
|
||||
for match in self._results:
|
||||
if match.fingerprint == item.fingerprint and match.chaos == item.chaos:
|
||||
match.add_submatch(item)
|
||||
return
|
||||
self._results.append(item)
|
||||
self._results = sorted(self._results)
|
||||
|
||||
def best(self) -> CharsetMatch | None:
|
||||
"""
|
||||
Simply return the first match. Strict equivalent to matches[0].
|
||||
"""
|
||||
if not self._results:
|
||||
return None
|
||||
return self._results[0]
|
||||
|
||||
def first(self) -> CharsetMatch | None:
|
||||
"""
|
||||
Redundant method, call the method best(). Kept for BC reasons.
|
||||
"""
|
||||
return self.best()
|
||||
|
||||
|
||||
CoherenceMatch = Tuple[str, float]
|
||||
CoherenceMatches = List[CoherenceMatch]
|
||||
|
||||
|
||||
class CliDetectionResult:
|
||||
def __init__(
|
||||
self,
|
||||
path: str,
|
||||
encoding: str | None,
|
||||
encoding_aliases: list[str],
|
||||
alternative_encodings: list[str],
|
||||
language: str,
|
||||
alphabets: list[str],
|
||||
has_sig_or_bom: bool,
|
||||
chaos: float,
|
||||
coherence: float,
|
||||
unicode_path: str | None,
|
||||
is_preferred: bool,
|
||||
):
|
||||
self.path: str = path
|
||||
self.unicode_path: str | None = unicode_path
|
||||
self.encoding: str | None = encoding
|
||||
self.encoding_aliases: list[str] = encoding_aliases
|
||||
self.alternative_encodings: list[str] = alternative_encodings
|
||||
self.language: str = language
|
||||
self.alphabets: list[str] = alphabets
|
||||
self.has_sig_or_bom: bool = has_sig_or_bom
|
||||
self.chaos: float = chaos
|
||||
self.coherence: float = coherence
|
||||
self.is_preferred: bool = is_preferred
|
||||
|
||||
@property
|
||||
def __dict__(self) -> dict[str, Any]: # type: ignore
|
||||
return {
|
||||
"path": self.path,
|
||||
"encoding": self.encoding,
|
||||
"encoding_aliases": self.encoding_aliases,
|
||||
"alternative_encodings": self.alternative_encodings,
|
||||
"language": self.language,
|
||||
"alphabets": self.alphabets,
|
||||
"has_sig_or_bom": self.has_sig_or_bom,
|
||||
"chaos": self.chaos,
|
||||
"coherence": self.coherence,
|
||||
"unicode_path": self.unicode_path,
|
||||
"is_preferred": self.is_preferred,
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
return dumps(self.__dict__, ensure_ascii=True, indent=4)
|
||||
@@ -0,0 +1,414 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import unicodedata
|
||||
from codecs import IncrementalDecoder
|
||||
from encodings.aliases import aliases
|
||||
from functools import lru_cache
|
||||
from re import findall
|
||||
from typing import Generator
|
||||
|
||||
from _multibytecodec import ( # type: ignore[import-not-found,import]
|
||||
MultibyteIncrementalDecoder,
|
||||
)
|
||||
|
||||
from .constant import (
|
||||
ENCODING_MARKS,
|
||||
IANA_SUPPORTED_SIMILAR,
|
||||
RE_POSSIBLE_ENCODING_INDICATION,
|
||||
UNICODE_RANGES_COMBINED,
|
||||
UNICODE_SECONDARY_RANGE_KEYWORD,
|
||||
UTF8_MAXIMAL_ALLOCATION,
|
||||
COMMON_CJK_CHARACTERS,
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_accentuated(character: str) -> bool:
|
||||
try:
|
||||
description: str = unicodedata.name(character)
|
||||
except ValueError: # Defensive: unicode database outdated?
|
||||
return False
|
||||
return (
|
||||
"WITH GRAVE" in description
|
||||
or "WITH ACUTE" in description
|
||||
or "WITH CEDILLA" in description
|
||||
or "WITH DIAERESIS" in description
|
||||
or "WITH CIRCUMFLEX" in description
|
||||
or "WITH TILDE" in description
|
||||
or "WITH MACRON" in description
|
||||
or "WITH RING ABOVE" in description
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def remove_accent(character: str) -> str:
|
||||
decomposed: str = unicodedata.decomposition(character)
|
||||
if not decomposed:
|
||||
return character
|
||||
|
||||
codes: list[str] = decomposed.split(" ")
|
||||
|
||||
return chr(int(codes[0], 16))
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def unicode_range(character: str) -> str | None:
|
||||
"""
|
||||
Retrieve the Unicode range official name from a single character.
|
||||
"""
|
||||
character_ord: int = ord(character)
|
||||
|
||||
for range_name, ord_range in UNICODE_RANGES_COMBINED.items():
|
||||
if character_ord in ord_range:
|
||||
return range_name
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_latin(character: str) -> bool:
|
||||
try:
|
||||
description: str = unicodedata.name(character)
|
||||
except ValueError: # Defensive: unicode database outdated?
|
||||
return False
|
||||
return "LATIN" in description
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_punctuation(character: str) -> bool:
|
||||
character_category: str = unicodedata.category(character)
|
||||
|
||||
if "P" in character_category:
|
||||
return True
|
||||
|
||||
character_range: str | None = unicode_range(character)
|
||||
|
||||
if character_range is None:
|
||||
return False
|
||||
|
||||
return "Punctuation" in character_range
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_symbol(character: str) -> bool:
|
||||
character_category: str = unicodedata.category(character)
|
||||
|
||||
if "S" in character_category or "N" in character_category:
|
||||
return True
|
||||
|
||||
character_range: str | None = unicode_range(character)
|
||||
|
||||
if character_range is None:
|
||||
return False
|
||||
|
||||
return "Forms" in character_range and character_category != "Lo"
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_emoticon(character: str) -> bool:
|
||||
character_range: str | None = unicode_range(character)
|
||||
|
||||
if character_range is None:
|
||||
return False
|
||||
|
||||
return "Emoticons" in character_range or "Pictographs" in character_range
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_separator(character: str) -> bool:
|
||||
if character.isspace() or character in {"|", "+", "<", ">"}:
|
||||
return True
|
||||
|
||||
character_category: str = unicodedata.category(character)
|
||||
|
||||
return "Z" in character_category or character_category in {"Po", "Pd", "Pc"}
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_case_variable(character: str) -> bool:
|
||||
return character.islower() != character.isupper()
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_cjk(character: str) -> bool:
|
||||
try:
|
||||
character_name = unicodedata.name(character)
|
||||
except ValueError: # Defensive: unicode database outdated?
|
||||
return False
|
||||
|
||||
return "CJK" in character_name
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_hiragana(character: str) -> bool:
|
||||
try:
|
||||
character_name = unicodedata.name(character)
|
||||
except ValueError: # Defensive: unicode database outdated?
|
||||
return False
|
||||
|
||||
return "HIRAGANA" in character_name
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_katakana(character: str) -> bool:
|
||||
try:
|
||||
character_name = unicodedata.name(character)
|
||||
except ValueError: # Defensive: unicode database outdated?
|
||||
return False
|
||||
|
||||
return "KATAKANA" in character_name
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_hangul(character: str) -> bool:
|
||||
try:
|
||||
character_name = unicodedata.name(character)
|
||||
except ValueError: # Defensive: unicode database outdated?
|
||||
return False
|
||||
|
||||
return "HANGUL" in character_name
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_thai(character: str) -> bool:
|
||||
try:
|
||||
character_name = unicodedata.name(character)
|
||||
except ValueError: # Defensive: unicode database outdated?
|
||||
return False
|
||||
|
||||
return "THAI" in character_name
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_arabic(character: str) -> bool:
|
||||
try:
|
||||
character_name = unicodedata.name(character)
|
||||
except ValueError: # Defensive: unicode database outdated?
|
||||
return False
|
||||
|
||||
return "ARABIC" in character_name
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_arabic_isolated_form(character: str) -> bool:
|
||||
try:
|
||||
character_name = unicodedata.name(character)
|
||||
except ValueError: # Defensive: unicode database outdated?
|
||||
return False
|
||||
|
||||
return "ARABIC" in character_name and "ISOLATED FORM" in character_name
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_cjk_uncommon(character: str) -> bool:
|
||||
return character not in COMMON_CJK_CHARACTERS
|
||||
|
||||
|
||||
@lru_cache(maxsize=len(UNICODE_RANGES_COMBINED))
|
||||
def is_unicode_range_secondary(range_name: str) -> bool:
|
||||
return any(keyword in range_name for keyword in UNICODE_SECONDARY_RANGE_KEYWORD)
|
||||
|
||||
|
||||
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
|
||||
def is_unprintable(character: str) -> bool:
|
||||
return (
|
||||
character.isspace() is False # includes \n \t \r \v
|
||||
and character.isprintable() is False
|
||||
and character != "\x1a" # Why? Its the ASCII substitute character.
|
||||
and character != "\ufeff" # bug discovered in Python,
|
||||
# Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space.
|
||||
)
|
||||
|
||||
|
||||
def any_specified_encoding(sequence: bytes, search_zone: int = 8192) -> str | None:
|
||||
"""
|
||||
Extract using ASCII-only decoder any specified encoding in the first n-bytes.
|
||||
"""
|
||||
if not isinstance(sequence, bytes):
|
||||
raise TypeError
|
||||
|
||||
seq_len: int = len(sequence)
|
||||
|
||||
results: list[str] = findall(
|
||||
RE_POSSIBLE_ENCODING_INDICATION,
|
||||
sequence[: min(seq_len, search_zone)].decode("ascii", errors="ignore"),
|
||||
)
|
||||
|
||||
if len(results) == 0:
|
||||
return None
|
||||
|
||||
for specified_encoding in results:
|
||||
specified_encoding = specified_encoding.lower().replace("-", "_")
|
||||
|
||||
encoding_alias: str
|
||||
encoding_iana: str
|
||||
|
||||
for encoding_alias, encoding_iana in aliases.items():
|
||||
if encoding_alias == specified_encoding:
|
||||
return encoding_iana
|
||||
if encoding_iana == specified_encoding:
|
||||
return encoding_iana
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def is_multi_byte_encoding(name: str) -> bool:
|
||||
"""
|
||||
Verify is a specific encoding is a multi byte one based on it IANA name
|
||||
"""
|
||||
return name in {
|
||||
"utf_8",
|
||||
"utf_8_sig",
|
||||
"utf_16",
|
||||
"utf_16_be",
|
||||
"utf_16_le",
|
||||
"utf_32",
|
||||
"utf_32_le",
|
||||
"utf_32_be",
|
||||
"utf_7",
|
||||
} or issubclass(
|
||||
importlib.import_module(f"encodings.{name}").IncrementalDecoder,
|
||||
MultibyteIncrementalDecoder,
|
||||
)
|
||||
|
||||
|
||||
def identify_sig_or_bom(sequence: bytes) -> tuple[str | None, bytes]:
|
||||
"""
|
||||
Identify and extract SIG/BOM in given sequence.
|
||||
"""
|
||||
|
||||
for iana_encoding in ENCODING_MARKS:
|
||||
marks: bytes | list[bytes] = ENCODING_MARKS[iana_encoding]
|
||||
|
||||
if isinstance(marks, bytes):
|
||||
marks = [marks]
|
||||
|
||||
for mark in marks:
|
||||
if sequence.startswith(mark):
|
||||
return iana_encoding, mark
|
||||
|
||||
return None, b""
|
||||
|
||||
|
||||
def should_strip_sig_or_bom(iana_encoding: str) -> bool:
|
||||
return iana_encoding not in {"utf_16", "utf_32"}
|
||||
|
||||
|
||||
def iana_name(cp_name: str, strict: bool = True) -> str:
|
||||
"""Returns the Python normalized encoding name (Not the IANA official name)."""
|
||||
cp_name = cp_name.lower().replace("-", "_")
|
||||
|
||||
encoding_alias: str
|
||||
encoding_iana: str
|
||||
|
||||
for encoding_alias, encoding_iana in aliases.items():
|
||||
if cp_name in [encoding_alias, encoding_iana]:
|
||||
return encoding_iana
|
||||
|
||||
if strict:
|
||||
raise ValueError(f"Unable to retrieve IANA for '{cp_name}'")
|
||||
|
||||
return cp_name
|
||||
|
||||
|
||||
def cp_similarity(iana_name_a: str, iana_name_b: str) -> float:
|
||||
if is_multi_byte_encoding(iana_name_a) or is_multi_byte_encoding(iana_name_b):
|
||||
return 0.0
|
||||
|
||||
decoder_a = importlib.import_module(f"encodings.{iana_name_a}").IncrementalDecoder
|
||||
decoder_b = importlib.import_module(f"encodings.{iana_name_b}").IncrementalDecoder
|
||||
|
||||
id_a: IncrementalDecoder = decoder_a(errors="ignore")
|
||||
id_b: IncrementalDecoder = decoder_b(errors="ignore")
|
||||
|
||||
character_match_count: int = 0
|
||||
|
||||
for i in range(255):
|
||||
to_be_decoded: bytes = bytes([i])
|
||||
if id_a.decode(to_be_decoded) == id_b.decode(to_be_decoded):
|
||||
character_match_count += 1
|
||||
|
||||
return character_match_count / 254
|
||||
|
||||
|
||||
def is_cp_similar(iana_name_a: str, iana_name_b: str) -> bool:
|
||||
"""
|
||||
Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using
|
||||
the function cp_similarity.
|
||||
"""
|
||||
return (
|
||||
iana_name_a in IANA_SUPPORTED_SIMILAR
|
||||
and iana_name_b in IANA_SUPPORTED_SIMILAR[iana_name_a]
|
||||
)
|
||||
|
||||
|
||||
def set_logging_handler(
|
||||
name: str = "charset_normalizer",
|
||||
level: int = logging.INFO,
|
||||
format_string: str = "%(asctime)s | %(levelname)s | %(message)s",
|
||||
) -> None:
|
||||
logger = logging.getLogger(name)
|
||||
logger.setLevel(level)
|
||||
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(logging.Formatter(format_string))
|
||||
logger.addHandler(handler)
|
||||
|
||||
|
||||
def cut_sequence_chunks(
|
||||
sequences: bytes,
|
||||
encoding_iana: str,
|
||||
offsets: range,
|
||||
chunk_size: int,
|
||||
bom_or_sig_available: bool,
|
||||
strip_sig_or_bom: bool,
|
||||
sig_payload: bytes,
|
||||
is_multi_byte_decoder: bool,
|
||||
decoded_payload: str | None = None,
|
||||
) -> Generator[str, None, None]:
|
||||
if decoded_payload and is_multi_byte_decoder is False:
|
||||
for i in offsets:
|
||||
chunk = decoded_payload[i : i + chunk_size]
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
else:
|
||||
for i in offsets:
|
||||
chunk_end = i + chunk_size
|
||||
if chunk_end > len(sequences) + 8:
|
||||
continue
|
||||
|
||||
cut_sequence = sequences[i : i + chunk_size]
|
||||
|
||||
if bom_or_sig_available and strip_sig_or_bom is False:
|
||||
cut_sequence = sig_payload + cut_sequence
|
||||
|
||||
chunk = cut_sequence.decode(
|
||||
encoding_iana,
|
||||
errors="ignore" if is_multi_byte_decoder else "strict",
|
||||
)
|
||||
|
||||
# multi-byte bad cutting detector and adjustment
|
||||
# not the cleanest way to perform that fix but clever enough for now.
|
||||
if is_multi_byte_decoder and i > 0:
|
||||
chunk_partial_size_chk: int = min(chunk_size, 16)
|
||||
|
||||
if (
|
||||
decoded_payload
|
||||
and chunk[:chunk_partial_size_chk] not in decoded_payload
|
||||
):
|
||||
for j in range(i, i - 4, -1):
|
||||
cut_sequence = sequences[j:chunk_end]
|
||||
|
||||
if bom_or_sig_available and strip_sig_or_bom is False:
|
||||
cut_sequence = sig_payload + cut_sequence
|
||||
|
||||
chunk = cut_sequence.decode(encoding_iana, errors="ignore")
|
||||
|
||||
if chunk[:chunk_partial_size_chk] in decoded_payload:
|
||||
break
|
||||
|
||||
yield chunk
|
||||
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
Expose version
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__version__ = "3.4.4"
|
||||
VERSION = __version__.split(".")
|
||||
@@ -0,0 +1 @@
|
||||
pip
|
||||
@@ -0,0 +1,122 @@
|
||||
Metadata-Version: 2.3
|
||||
Name: docker
|
||||
Version: 7.1.0
|
||||
Summary: A Python library for the Docker Engine API.
|
||||
Project-URL: Changelog, https://docker-py.readthedocs.io/en/stable/change-log.html
|
||||
Project-URL: Documentation, https://docker-py.readthedocs.io
|
||||
Project-URL: Homepage, https://github.com/docker/docker-py
|
||||
Project-URL: Source, https://github.com/docker/docker-py
|
||||
Project-URL: Tracker, https://github.com/docker/docker-py/issues
|
||||
Maintainer-email: "Docker Inc." <no-reply@docker.com>
|
||||
License-Expression: Apache-2.0
|
||||
License-File: LICENSE
|
||||
Classifier: Development Status :: 5 - Production/Stable
|
||||
Classifier: Environment :: Other Environment
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: License :: OSI Approved :: Apache Software License
|
||||
Classifier: Operating System :: OS Independent
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3.8
|
||||
Classifier: Programming Language :: Python :: 3.9
|
||||
Classifier: Programming Language :: Python :: 3.10
|
||||
Classifier: Programming Language :: Python :: 3.11
|
||||
Classifier: Programming Language :: Python :: 3.12
|
||||
Classifier: Topic :: Software Development
|
||||
Classifier: Topic :: Utilities
|
||||
Requires-Python: >=3.8
|
||||
Requires-Dist: pywin32>=304; sys_platform == 'win32'
|
||||
Requires-Dist: requests>=2.26.0
|
||||
Requires-Dist: urllib3>=1.26.0
|
||||
Provides-Extra: dev
|
||||
Requires-Dist: coverage==7.2.7; extra == 'dev'
|
||||
Requires-Dist: pytest-cov==4.1.0; extra == 'dev'
|
||||
Requires-Dist: pytest-timeout==2.1.0; extra == 'dev'
|
||||
Requires-Dist: pytest==7.4.2; extra == 'dev'
|
||||
Requires-Dist: ruff==0.1.8; extra == 'dev'
|
||||
Provides-Extra: docs
|
||||
Requires-Dist: myst-parser==0.18.0; extra == 'docs'
|
||||
Requires-Dist: sphinx==5.1.1; extra == 'docs'
|
||||
Provides-Extra: ssh
|
||||
Requires-Dist: paramiko>=2.4.3; extra == 'ssh'
|
||||
Provides-Extra: tls
|
||||
Provides-Extra: websockets
|
||||
Requires-Dist: websocket-client>=1.3.0; extra == 'websockets'
|
||||
Description-Content-Type: text/markdown
|
||||
|
||||
# Docker SDK for Python
|
||||
|
||||
[](https://github.com/docker/docker-py/actions/workflows/ci.yml)
|
||||
|
||||
A Python library for the Docker Engine API. It lets you do anything the `docker` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc.
|
||||
|
||||
## Installation
|
||||
|
||||
The latest stable version [is available on PyPI](https://pypi.python.org/pypi/docker/). Install with pip:
|
||||
|
||||
pip install docker
|
||||
|
||||
> Older versions (< 6.0) required installing `docker[tls]` for SSL/TLS support.
|
||||
> This is no longer necessary and is a no-op, but is supported for backwards compatibility.
|
||||
|
||||
## Usage
|
||||
|
||||
Connect to Docker using the default socket or the configuration in your environment:
|
||||
|
||||
```python
|
||||
import docker
|
||||
client = docker.from_env()
|
||||
```
|
||||
|
||||
You can run containers:
|
||||
|
||||
```python
|
||||
>>> client.containers.run("ubuntu:latest", "echo hello world")
|
||||
'hello world\n'
|
||||
```
|
||||
|
||||
You can run containers in the background:
|
||||
|
||||
```python
|
||||
>>> client.containers.run("bfirsh/reticulate-splines", detach=True)
|
||||
<Container '45e6d2de7c54'>
|
||||
```
|
||||
|
||||
You can manage containers:
|
||||
|
||||
```python
|
||||
>>> client.containers.list()
|
||||
[<Container '45e6d2de7c54'>, <Container 'db18e4f20eaa'>, ...]
|
||||
|
||||
>>> container = client.containers.get('45e6d2de7c54')
|
||||
|
||||
>>> container.attrs['Config']['Image']
|
||||
"bfirsh/reticulate-splines"
|
||||
|
||||
>>> container.logs()
|
||||
"Reticulating spline 1...\n"
|
||||
|
||||
>>> container.stop()
|
||||
```
|
||||
|
||||
You can stream logs:
|
||||
|
||||
```python
|
||||
>>> for line in container.logs(stream=True):
|
||||
... print(line.strip())
|
||||
Reticulating spline 2...
|
||||
Reticulating spline 3...
|
||||
...
|
||||
```
|
||||
|
||||
You can manage images:
|
||||
|
||||
```python
|
||||
>>> client.images.pull('nginx')
|
||||
<Image 'nginx'>
|
||||
|
||||
>>> client.images.list()
|
||||
[<Image 'ubuntu'>, <Image 'nginx'>, ...]
|
||||
```
|
||||
|
||||
[Read the full documentation](https://docker-py.readthedocs.io) to see everything you can do.
|
||||
@@ -0,0 +1,140 @@
|
||||
docker-7.1.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
|
||||
docker-7.1.0.dist-info/METADATA,sha256=qlCRMTVMhL1JtSk-Mkb4lhXlIzmUcU3JSbRkz64kbQs,3785
|
||||
docker-7.1.0.dist-info/RECORD,,
|
||||
docker-7.1.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||
docker-7.1.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
|
||||
docker-7.1.0.dist-info/licenses/LICENSE,sha256=8vCwf6XkksEdJ6oNLz8aDmS50X8y2KpImuKvlgmvM7I,10758
|
||||
docker/__init__.py,sha256=ER6EbOtcG0e3V4cTDGF7kQvsNDFKAbh2PMNN6lveWhw,193
|
||||
docker/__pycache__/__init__.cpython-313.pyc,,
|
||||
docker/__pycache__/_version.cpython-313.pyc,,
|
||||
docker/__pycache__/auth.cpython-313.pyc,,
|
||||
docker/__pycache__/client.cpython-313.pyc,,
|
||||
docker/__pycache__/constants.cpython-313.pyc,,
|
||||
docker/__pycache__/errors.cpython-313.pyc,,
|
||||
docker/__pycache__/tls.cpython-313.pyc,,
|
||||
docker/__pycache__/version.cpython-313.pyc,,
|
||||
docker/_version.py,sha256=jXWXDdqcry2vyc742uYcr4NaI1j-6TGMyveJUqhZf9Y,411
|
||||
docker/api/__init__.py,sha256=v0_aftqetK9PoSKyyKXlvQXh2dDdIDygDku8UtYqR-k,30
|
||||
docker/api/__pycache__/__init__.cpython-313.pyc,,
|
||||
docker/api/__pycache__/build.cpython-313.pyc,,
|
||||
docker/api/__pycache__/client.cpython-313.pyc,,
|
||||
docker/api/__pycache__/config.cpython-313.pyc,,
|
||||
docker/api/__pycache__/container.cpython-313.pyc,,
|
||||
docker/api/__pycache__/daemon.cpython-313.pyc,,
|
||||
docker/api/__pycache__/exec_api.cpython-313.pyc,,
|
||||
docker/api/__pycache__/image.cpython-313.pyc,,
|
||||
docker/api/__pycache__/network.cpython-313.pyc,,
|
||||
docker/api/__pycache__/plugin.cpython-313.pyc,,
|
||||
docker/api/__pycache__/secret.cpython-313.pyc,,
|
||||
docker/api/__pycache__/service.cpython-313.pyc,,
|
||||
docker/api/__pycache__/swarm.cpython-313.pyc,,
|
||||
docker/api/__pycache__/volume.cpython-313.pyc,,
|
||||
docker/api/build.py,sha256=C3HPhJfforYYDUWkNIACagdvf77Fj9DCC5EmIY3T-JA,16063
|
||||
docker/api/client.py,sha256=FULUhlBCONJRpv1mi7hOuKVvP38uPRX7ui7kA6i8tVA,19429
|
||||
docker/api/config.py,sha256=B1xH9KGgMFGJ9oz-R7IvlW5FUWuFxXDkNnM5YfgPDrQ,2706
|
||||
docker/api/container.py,sha256=NCncR2DUx6MOBxgk1v02_4155MDrzRONWR3NzKZ-W4U,52842
|
||||
docker/api/daemon.py,sha256=QHMYZH0w4xhOeWWDyr5ae13ktKu8-tnG7U44dWl9PxQ,6008
|
||||
docker/api/exec_api.py,sha256=rFVsjXPldQiwKgrlLOwhtFOaW6W8nneZZN36ANl7rHU,6210
|
||||
docker/api/image.py,sha256=TmxRX8GJkxWa-E43-00PHXGZmEvnjKVBfbH5R8FD0MA,20256
|
||||
docker/api/network.py,sha256=pZgGHtdnOHI4-LW47LB_Z9TNwX4b4KdS-08kbFzVUQk,10672
|
||||
docker/api/plugin.py,sha256=eR6GI12ECXg_vLrQ9GBgMuiUvtcqmDxzhmw59ymOVZw,8982
|
||||
docker/api/secret.py,sha256=qp8Zrg6l6Q8Dl4JtJfem1r90s-gaHWKjmS1ctrSlv3g,2869
|
||||
docker/api/service.py,sha256=bJs2jf_Io6l3_Jqsui6lW4-bhQvh9NUhj-wZQnOTyPI,19215
|
||||
docker/api/swarm.py,sha256=Wc8QrhwOMc4vBSx4i5mQrg47NVEjLs8qpv5NmNHGRdo,18089
|
||||
docker/api/volume.py,sha256=Hyaz7ZejJAKwLMAUiM21jYbWlVy-FAOzpL9p5vuXQmc,5119
|
||||
docker/auth.py,sha256=8HuFmbfDBb9N8AiRrQJtvABt4nFOEk4F-UZw9vP7IGU,12977
|
||||
docker/client.py,sha256=z2V8T0WIjDSU395FV821mTke1JQiy2bcc6_QvwnoUZk,7704
|
||||
docker/constants.py,sha256=nYtw3W7m-Gt607bUieu-1hLc_Rp6vBTwk_CvU8KvfCw,1195
|
||||
docker/context/__init__.py,sha256=iTtaBZOZr55DCYvrqS3snBjSCLzAW4e0kHDQoNqGzak,57
|
||||
docker/context/__pycache__/__init__.cpython-313.pyc,,
|
||||
docker/context/__pycache__/api.cpython-313.pyc,,
|
||||
docker/context/__pycache__/config.cpython-313.pyc,,
|
||||
docker/context/__pycache__/context.cpython-313.pyc,,
|
||||
docker/context/api.py,sha256=CaoT-NvsbQWTQogm5ooDM8MjDgEsQ190u05s6pgSvsg,6326
|
||||
docker/context/config.py,sha256=3Ak2ou2ZhsFOKcyjQQrMLENxPE44-NVJ97EM7Vkebqo,2151
|
||||
docker/context/context.py,sha256=dzrJdsF_J1lYgfFkMPWVwGT_1urt9eWu0DYmpYMbjC0,7828
|
||||
docker/credentials/__init__.py,sha256=gCzqshZEPYhTVnqVWq8reBY2bGpoj16T5CHFId0YGKw,197
|
||||
docker/credentials/__pycache__/__init__.cpython-313.pyc,,
|
||||
docker/credentials/__pycache__/constants.cpython-313.pyc,,
|
||||
docker/credentials/__pycache__/errors.cpython-313.pyc,,
|
||||
docker/credentials/__pycache__/store.cpython-313.pyc,,
|
||||
docker/credentials/__pycache__/utils.cpython-313.pyc,,
|
||||
docker/credentials/constants.py,sha256=5HjRGorpVvkamvLBI9yLPvur1E0glaq_ZhuRfbVA_bE,142
|
||||
docker/credentials/errors.py,sha256=FHXYAfHWD1CzAlxRDX3bP1D2BRJ48YhfVFtrPgBeHKk,440
|
||||
docker/credentials/store.py,sha256=8VjTXhi9wUZPdSO_MCt8dGdgnFVxSTGNsZ5P3AUaH04,3262
|
||||
docker/credentials/utils.py,sha256=0xhfjlCxa2_CyRGCSKzPvuAJzV-Ty7t-O5MKnyEq1us,224
|
||||
docker/errors.py,sha256=p5OV-0CsrQmLkGsGyMRV9nHlLip3uwXNZbOWDI6sfrs,5340
|
||||
docker/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||
docker/models/__pycache__/__init__.cpython-313.pyc,,
|
||||
docker/models/__pycache__/configs.cpython-313.pyc,,
|
||||
docker/models/__pycache__/containers.cpython-313.pyc,,
|
||||
docker/models/__pycache__/images.cpython-313.pyc,,
|
||||
docker/models/__pycache__/networks.cpython-313.pyc,,
|
||||
docker/models/__pycache__/nodes.cpython-313.pyc,,
|
||||
docker/models/__pycache__/plugins.cpython-313.pyc,,
|
||||
docker/models/__pycache__/resource.cpython-313.pyc,,
|
||||
docker/models/__pycache__/secrets.cpython-313.pyc,,
|
||||
docker/models/__pycache__/services.cpython-313.pyc,,
|
||||
docker/models/__pycache__/swarm.cpython-313.pyc,,
|
||||
docker/models/__pycache__/volumes.cpython-313.pyc,,
|
||||
docker/models/configs.py,sha256=Nt1itqz5AXrqrhuYKY16-l7AELELSB0wf6ApQAwGtRI,1845
|
||||
docker/models/containers.py,sha256=bhzBwiwUtBpxzgFOfFwEhrUl8CmJkYs7t6xAG6f9w-A,46735
|
||||
docker/models/images.py,sha256=aDN_yP8adh8Ccu_kszGiFMvLFJX12AbaJdzTJFpEw2A,18000
|
||||
docker/models/networks.py,sha256=ksxKWpPuIt-2Sw248zS_uLa8GzokJiMnGOua4LgLRUw,7962
|
||||
docker/models/nodes.py,sha256=-E8KscpxlOu73MclTcyo1b9eHPt0Z84ov9e-yXp3RgE,2928
|
||||
docker/models/plugins.py,sha256=xxaaAEsDyxBUNSOzMa-xNLanMismQL_eOfsgw-QCtrc,5971
|
||||
docker/models/resource.py,sha256=JkXvJ9cuQ7f-fu4EwNsFuV6nCq5FcgtoqWOqodOTPZM,2580
|
||||
docker/models/secrets.py,sha256=Cn0baKmg-h--hf68rzbMNh7H-BxLcj8sNLZBJLoI_MQ,1845
|
||||
docker/models/services.py,sha256=V749QHEgydxwpCajz5Lgw2O7L9LvkAOy1OpfmpYENTo,13924
|
||||
docker/models/swarm.py,sha256=5QG3rdr8uwfdAH59-Tyw32v4cb6Gfv8GlFYwHsmItA0,8290
|
||||
docker/models/volumes.py,sha256=Wi_YyINASJxWyFTDbLv_5EQjzMX_sRslPfcbNPHT1-U,2848
|
||||
docker/tls.py,sha256=nrCLgBWW6MPyJCQ_tzUWSO1xbZkJ7ZYMrbQbo9jOXls,2320
|
||||
docker/transport/__init__.py,sha256=KbwVo0ISUASDTyCbCo0jcmoki1kfXarOqxrC9uqYUmE,233
|
||||
docker/transport/__pycache__/__init__.cpython-313.pyc,,
|
||||
docker/transport/__pycache__/basehttpadapter.cpython-313.pyc,,
|
||||
docker/transport/__pycache__/npipeconn.cpython-313.pyc,,
|
||||
docker/transport/__pycache__/npipesocket.cpython-313.pyc,,
|
||||
docker/transport/__pycache__/sshconn.cpython-313.pyc,,
|
||||
docker/transport/__pycache__/unixconn.cpython-313.pyc,,
|
||||
docker/transport/basehttpadapter.py,sha256=Qvgc6IObkWicKc0_eTbmsLo2nAlY4gmUhndTqHTNJjc,457
|
||||
docker/transport/npipeconn.py,sha256=fyrWFxCokHNzCtrT-lFFzEEolIiGaNni_puBEKxveew,3525
|
||||
docker/transport/npipesocket.py,sha256=ctJN4Vq_331l7ItLDACOKCYOkbiXxrmTaQjqOPX02wQ,6585
|
||||
docker/transport/sshconn.py,sha256=QoK1n7RNuSrCObJJsQFHC37wAfHMbGiNtibj-zh72og,7923
|
||||
docker/transport/unixconn.py,sha256=FMROVi_7WbHX5vGiJqsUR3s7arOj3-I2d202bWbKZNM,2915
|
||||
docker/types/__init__.py,sha256=vMkPvrD90jN6Tja2oPwNK1dzvhiwdJGuireai-dE3_g,625
|
||||
docker/types/__pycache__/__init__.cpython-313.pyc,,
|
||||
docker/types/__pycache__/base.cpython-313.pyc,,
|
||||
docker/types/__pycache__/containers.cpython-313.pyc,,
|
||||
docker/types/__pycache__/daemon.cpython-313.pyc,,
|
||||
docker/types/__pycache__/healthcheck.cpython-313.pyc,,
|
||||
docker/types/__pycache__/networks.cpython-313.pyc,,
|
||||
docker/types/__pycache__/services.cpython-313.pyc,,
|
||||
docker/types/__pycache__/swarm.cpython-313.pyc,,
|
||||
docker/types/base.py,sha256=shDFz5VEx-mfdtMtJQQIpT8GG6rZ-NmfUS6uFvSr3ig,110
|
||||
docker/types/containers.py,sha256=YDNMMqMd6lvjF2j8WT4gA4nQRmtAziJDypXezD1qk9g,27412
|
||||
docker/types/daemon.py,sha256=nR3-Hz9H04I1GecSateTXLwbq_ivA86QgsX8q8a7Lu0,1943
|
||||
docker/types/healthcheck.py,sha256=leuqfN6oUwlxSd_iMGFPfRiTB2ZipzESHtBj8NxzVv4,2776
|
||||
docker/types/networks.py,sha256=7I82VYPQd7Rjqfwx0-Q2jOIwGK-K8UiS0K61YkZtuuw,4240
|
||||
docker/types/services.py,sha256=3o2qwd_ACI1uwK0ClgxNg8RWictwQq6FYVnVauEinGc,33151
|
||||
docker/types/swarm.py,sha256=-xhjuOgXBE3Ct9zsuD0r-k8jROL-XETmRTWlRgGZTG4,4653
|
||||
docker/utils/__init__.py,sha256=M1IRXWTzd2uihzlUtW0vLvwRX3-C0dyxG-1JnkBnGaw,664
|
||||
docker/utils/__pycache__/__init__.cpython-313.pyc,,
|
||||
docker/utils/__pycache__/build.cpython-313.pyc,,
|
||||
docker/utils/__pycache__/config.cpython-313.pyc,,
|
||||
docker/utils/__pycache__/decorators.cpython-313.pyc,,
|
||||
docker/utils/__pycache__/fnmatch.cpython-313.pyc,,
|
||||
docker/utils/__pycache__/json_stream.cpython-313.pyc,,
|
||||
docker/utils/__pycache__/ports.cpython-313.pyc,,
|
||||
docker/utils/__pycache__/proxy.cpython-313.pyc,,
|
||||
docker/utils/__pycache__/socket.cpython-313.pyc,,
|
||||
docker/utils/__pycache__/utils.cpython-313.pyc,,
|
||||
docker/utils/build.py,sha256=3DO8EmP4uH7p_KoBsMvVWDDQYFvQ3a-2Zl19rRlNkzA,8266
|
||||
docker/utils/config.py,sha256=f5YJSiRJSGiOhwSTQJz_7HsA9AHSENib692TXNld6w0,1724
|
||||
docker/utils/decorators.py,sha256=72pAP26Qdv7FNTrtrEJn6lbOmNb92dPs-xwVvrH8Jy4,1492
|
||||
docker/utils/fnmatch.py,sha256=v6KNlQ9cHH9mL8bvOtKqbLwZe6wyVvWLLHZRxoJfstU,3295
|
||||
docker/utils/json_stream.py,sha256=HeSzhiw6t_oMaHOu_YTUNSvbHWK4F_A884pU7DqmcCg,2184
|
||||
docker/utils/ports.py,sha256=JKcWm4-5EwuM2fluvEXdZpgbjS9PHwPQzOBa-TTSwvM,2799
|
||||
docker/utils/proxy.py,sha256=4ZxhRGlPSuEb8xLEuLin2zxbQ8BCsCfmnxzfujGoazc,2246
|
||||
docker/utils/socket.py,sha256=JXr3rMl98PWHKEs1PN6wta9D-8y9I-w4Nbk5WwUwqQM,5073
|
||||
docker/utils/utils.py,sha256=P-qjbAicJk4VcJK6jyk4TnNbetPzyOLmar5ArGmKZ4Y,14124
|
||||
docker/version.py,sha256=7BJkAx-Ukdj2ZMkpwQL5UiiByEpcRSxeUr6tJZh03WM,240
|
||||
@@ -0,0 +1,4 @@
|
||||
Wheel-Version: 1.0
|
||||
Generator: hatchling 1.24.2
|
||||
Root-Is-Purelib: true
|
||||
Tag: py3-none-any
|
||||
@@ -0,0 +1,191 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2016 Docker, Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,7 @@
|
||||
from .api import APIClient
|
||||
from .client import DockerClient, from_env
|
||||
from .context import Context, ContextAPI
|
||||
from .tls import TLSConfig
|
||||
from .version import __version__
|
||||
|
||||
__title__ = 'docker'
|
||||
@@ -0,0 +1,16 @@
|
||||
# file generated by setuptools_scm
|
||||
# don't change, don't track in version control
|
||||
TYPE_CHECKING = False
|
||||
if TYPE_CHECKING:
|
||||
from typing import Tuple, Union
|
||||
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
||||
else:
|
||||
VERSION_TUPLE = object
|
||||
|
||||
version: str
|
||||
__version__: str
|
||||
__version_tuple__: VERSION_TUPLE
|
||||
version_tuple: VERSION_TUPLE
|
||||
|
||||
__version__ = version = '7.1.0'
|
||||
__version_tuple__ = version_tuple = (7, 1, 0)
|
||||
@@ -0,0 +1 @@
|
||||
from .client import APIClient
|
||||
@@ -0,0 +1,382 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
|
||||
from .. import auth, constants, errors, utils
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BuildApiMixin:
|
||||
def build(self, path=None, tag=None, quiet=False, fileobj=None,
|
||||
nocache=False, rm=False, timeout=None,
|
||||
custom_context=False, encoding=None, pull=False,
|
||||
forcerm=False, dockerfile=None, container_limits=None,
|
||||
decode=False, buildargs=None, gzip=False, shmsize=None,
|
||||
labels=None, cache_from=None, target=None, network_mode=None,
|
||||
squash=None, extra_hosts=None, platform=None, isolation=None,
|
||||
use_config_proxy=True):
|
||||
"""
|
||||
Similar to the ``docker build`` command. Either ``path`` or ``fileobj``
|
||||
needs to be set. ``path`` can be a local path (to a directory
|
||||
containing a Dockerfile) or a remote URL. ``fileobj`` must be a
|
||||
readable file-like object to a Dockerfile.
|
||||
|
||||
If you have a tar file for the Docker build context (including a
|
||||
Dockerfile) already, pass a readable file-like object to ``fileobj``
|
||||
and also pass ``custom_context=True``. If the stream is compressed
|
||||
also, set ``encoding`` to the correct value (e.g ``gzip``).
|
||||
|
||||
Example:
|
||||
>>> from io import BytesIO
|
||||
>>> from docker import APIClient
|
||||
>>> dockerfile = '''
|
||||
... # Shared Volume
|
||||
... FROM busybox:buildroot-2014.02
|
||||
... VOLUME /data
|
||||
... CMD ["/bin/sh"]
|
||||
... '''
|
||||
>>> f = BytesIO(dockerfile.encode('utf-8'))
|
||||
>>> cli = APIClient(base_url='tcp://127.0.0.1:2375')
|
||||
>>> response = [line for line in cli.build(
|
||||
... fileobj=f, rm=True, tag='yourname/volume'
|
||||
... )]
|
||||
>>> response
|
||||
['{"stream":" ---\\u003e a9eb17255234\\n"}',
|
||||
'{"stream":"Step 1 : VOLUME /data\\n"}',
|
||||
'{"stream":" ---\\u003e Running in abdc1e6896c6\\n"}',
|
||||
'{"stream":" ---\\u003e 713bca62012e\\n"}',
|
||||
'{"stream":"Removing intermediate container abdc1e6896c6\\n"}',
|
||||
'{"stream":"Step 2 : CMD [\\"/bin/sh\\"]\\n"}',
|
||||
'{"stream":" ---\\u003e Running in dba30f2a1a7e\\n"}',
|
||||
'{"stream":" ---\\u003e 032b8b2855fc\\n"}',
|
||||
'{"stream":"Removing intermediate container dba30f2a1a7e\\n"}',
|
||||
'{"stream":"Successfully built 032b8b2855fc\\n"}']
|
||||
|
||||
Args:
|
||||
path (str): Path to the directory containing the Dockerfile
|
||||
fileobj: A file object to use as the Dockerfile. (Or a file-like
|
||||
object)
|
||||
tag (str): A tag to add to the final image
|
||||
quiet (bool): Whether to return the status
|
||||
nocache (bool): Don't use the cache when set to ``True``
|
||||
rm (bool): Remove intermediate containers. The ``docker build``
|
||||
command now defaults to ``--rm=true``, but we have kept the old
|
||||
default of `False` to preserve backward compatibility
|
||||
timeout (int): HTTP timeout
|
||||
custom_context (bool): Optional if using ``fileobj``
|
||||
encoding (str): The encoding for a stream. Set to ``gzip`` for
|
||||
compressing
|
||||
pull (bool): Downloads any updates to the FROM image in Dockerfiles
|
||||
forcerm (bool): Always remove intermediate containers, even after
|
||||
unsuccessful builds
|
||||
dockerfile (str): path within the build context to the Dockerfile
|
||||
gzip (bool): If set to ``True``, gzip compression/encoding is used
|
||||
buildargs (dict): A dictionary of build arguments
|
||||
container_limits (dict): A dictionary of limits applied to each
|
||||
container created by the build process. Valid keys:
|
||||
|
||||
- memory (int): set memory limit for build
|
||||
- memswap (int): Total memory (memory + swap), -1 to disable
|
||||
swap
|
||||
- cpushares (int): CPU shares (relative weight)
|
||||
- cpusetcpus (str): CPUs in which to allow execution, e.g.,
|
||||
``"0-3"``, ``"0,1"``
|
||||
decode (bool): If set to ``True``, the returned stream will be
|
||||
decoded into dicts on the fly. Default ``False``
|
||||
shmsize (int): Size of `/dev/shm` in bytes. The size must be
|
||||
greater than 0. If omitted the system uses 64MB
|
||||
labels (dict): A dictionary of labels to set on the image
|
||||
cache_from (:py:class:`list`): A list of images used for build
|
||||
cache resolution
|
||||
target (str): Name of the build-stage to build in a multi-stage
|
||||
Dockerfile
|
||||
network_mode (str): networking mode for the run commands during
|
||||
build
|
||||
squash (bool): Squash the resulting images layers into a
|
||||
single layer.
|
||||
extra_hosts (dict): Extra hosts to add to /etc/hosts in building
|
||||
containers, as a mapping of hostname to IP address.
|
||||
platform (str): Platform in the format ``os[/arch[/variant]]``
|
||||
isolation (str): Isolation technology used during build.
|
||||
Default: `None`.
|
||||
use_config_proxy (bool): If ``True``, and if the docker client
|
||||
configuration file (``~/.docker/config.json`` by default)
|
||||
contains a proxy configuration, the corresponding environment
|
||||
variables will be set in the container being built.
|
||||
|
||||
Returns:
|
||||
A generator for the build output.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
``TypeError``
|
||||
If neither ``path`` nor ``fileobj`` is specified.
|
||||
"""
|
||||
remote = context = None
|
||||
headers = {}
|
||||
container_limits = container_limits or {}
|
||||
buildargs = buildargs or {}
|
||||
if path is None and fileobj is None:
|
||||
raise TypeError("Either path or fileobj needs to be provided.")
|
||||
if gzip and encoding is not None:
|
||||
raise errors.DockerException(
|
||||
'Can not use custom encoding if gzip is enabled'
|
||||
)
|
||||
if tag is not None:
|
||||
if not utils.match_tag(tag):
|
||||
raise errors.DockerException(
|
||||
f"invalid tag '{tag}': invalid reference format"
|
||||
)
|
||||
for key in container_limits.keys():
|
||||
if key not in constants.CONTAINER_LIMITS_KEYS:
|
||||
raise errors.DockerException(
|
||||
f"invalid tag '{tag}': invalid reference format"
|
||||
)
|
||||
if custom_context:
|
||||
if not fileobj:
|
||||
raise TypeError("You must specify fileobj with custom_context")
|
||||
context = fileobj
|
||||
elif fileobj is not None:
|
||||
context = utils.mkbuildcontext(fileobj)
|
||||
elif path.startswith(('http://', 'https://',
|
||||
'git://', 'github.com/', 'git@')):
|
||||
remote = path
|
||||
elif not os.path.isdir(path):
|
||||
raise TypeError("You must specify a directory to build in path")
|
||||
else:
|
||||
dockerignore = os.path.join(path, '.dockerignore')
|
||||
exclude = None
|
||||
if os.path.exists(dockerignore):
|
||||
with open(dockerignore) as f:
|
||||
exclude = list(filter(
|
||||
lambda x: x != '' and x[0] != '#',
|
||||
[line.strip() for line in f.read().splitlines()]
|
||||
))
|
||||
dockerfile = process_dockerfile(dockerfile, path)
|
||||
context = utils.tar(
|
||||
path, exclude=exclude, dockerfile=dockerfile, gzip=gzip
|
||||
)
|
||||
encoding = 'gzip' if gzip else encoding
|
||||
|
||||
u = self._url('/build')
|
||||
params = {
|
||||
't': tag,
|
||||
'remote': remote,
|
||||
'q': quiet,
|
||||
'nocache': nocache,
|
||||
'rm': rm,
|
||||
'forcerm': forcerm,
|
||||
'pull': pull,
|
||||
'dockerfile': dockerfile,
|
||||
}
|
||||
params.update(container_limits)
|
||||
|
||||
if use_config_proxy:
|
||||
proxy_args = self._proxy_configs.get_environment()
|
||||
for k, v in proxy_args.items():
|
||||
buildargs.setdefault(k, v)
|
||||
if buildargs:
|
||||
params.update({'buildargs': json.dumps(buildargs)})
|
||||
|
||||
if shmsize:
|
||||
if utils.version_gte(self._version, '1.22'):
|
||||
params.update({'shmsize': shmsize})
|
||||
else:
|
||||
raise errors.InvalidVersion(
|
||||
'shmsize was only introduced in API version 1.22'
|
||||
)
|
||||
|
||||
if labels:
|
||||
if utils.version_gte(self._version, '1.23'):
|
||||
params.update({'labels': json.dumps(labels)})
|
||||
else:
|
||||
raise errors.InvalidVersion(
|
||||
'labels was only introduced in API version 1.23'
|
||||
)
|
||||
|
||||
if cache_from:
|
||||
if utils.version_gte(self._version, '1.25'):
|
||||
params.update({'cachefrom': json.dumps(cache_from)})
|
||||
else:
|
||||
raise errors.InvalidVersion(
|
||||
'cache_from was only introduced in API version 1.25'
|
||||
)
|
||||
|
||||
if target:
|
||||
if utils.version_gte(self._version, '1.29'):
|
||||
params.update({'target': target})
|
||||
else:
|
||||
raise errors.InvalidVersion(
|
||||
'target was only introduced in API version 1.29'
|
||||
)
|
||||
|
||||
if network_mode:
|
||||
if utils.version_gte(self._version, '1.25'):
|
||||
params.update({'networkmode': network_mode})
|
||||
else:
|
||||
raise errors.InvalidVersion(
|
||||
'network_mode was only introduced in API version 1.25'
|
||||
)
|
||||
|
||||
if squash:
|
||||
if utils.version_gte(self._version, '1.25'):
|
||||
params.update({'squash': squash})
|
||||
else:
|
||||
raise errors.InvalidVersion(
|
||||
'squash was only introduced in API version 1.25'
|
||||
)
|
||||
|
||||
if extra_hosts is not None:
|
||||
if utils.version_lt(self._version, '1.27'):
|
||||
raise errors.InvalidVersion(
|
||||
'extra_hosts was only introduced in API version 1.27'
|
||||
)
|
||||
|
||||
if isinstance(extra_hosts, dict):
|
||||
extra_hosts = utils.format_extra_hosts(extra_hosts)
|
||||
params.update({'extrahosts': extra_hosts})
|
||||
|
||||
if platform is not None:
|
||||
if utils.version_lt(self._version, '1.32'):
|
||||
raise errors.InvalidVersion(
|
||||
'platform was only introduced in API version 1.32'
|
||||
)
|
||||
params['platform'] = platform
|
||||
|
||||
if isolation is not None:
|
||||
if utils.version_lt(self._version, '1.24'):
|
||||
raise errors.InvalidVersion(
|
||||
'isolation was only introduced in API version 1.24'
|
||||
)
|
||||
params['isolation'] = isolation
|
||||
|
||||
if context is not None:
|
||||
headers = {'Content-Type': 'application/tar'}
|
||||
if encoding:
|
||||
headers['Content-Encoding'] = encoding
|
||||
|
||||
self._set_auth_headers(headers)
|
||||
|
||||
response = self._post(
|
||||
u,
|
||||
data=context,
|
||||
params=params,
|
||||
headers=headers,
|
||||
stream=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if context is not None and not custom_context:
|
||||
context.close()
|
||||
|
||||
return self._stream_helper(response, decode=decode)
|
||||
|
||||
@utils.minimum_version('1.31')
|
||||
def prune_builds(self, filters=None, keep_storage=None, all=None):
|
||||
"""
|
||||
Delete the builder cache
|
||||
|
||||
Args:
|
||||
filters (dict): Filters to process on the prune list.
|
||||
Needs Docker API v1.39+
|
||||
Available filters:
|
||||
- dangling (bool): When set to true (or 1), prune only
|
||||
unused and untagged images.
|
||||
- until (str): Can be Unix timestamps, date formatted
|
||||
timestamps, or Go duration strings (e.g. 10m, 1h30m) computed
|
||||
relative to the daemon's local time.
|
||||
keep_storage (int): Amount of disk space in bytes to keep for cache.
|
||||
Needs Docker API v1.39+
|
||||
all (bool): Remove all types of build cache.
|
||||
Needs Docker API v1.39+
|
||||
|
||||
Returns:
|
||||
(dict): A dictionary containing information about the operation's
|
||||
result. The ``SpaceReclaimed`` key indicates the amount of
|
||||
bytes of disk space reclaimed.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
url = self._url("/build/prune")
|
||||
if (filters, keep_storage, all) != (None, None, None) \
|
||||
and utils.version_lt(self._version, '1.39'):
|
||||
raise errors.InvalidVersion(
|
||||
'`filters`, `keep_storage`, and `all` args are only available '
|
||||
'for API version > 1.38'
|
||||
)
|
||||
params = {}
|
||||
if filters is not None:
|
||||
params['filters'] = utils.convert_filters(filters)
|
||||
if keep_storage is not None:
|
||||
params['keep-storage'] = keep_storage
|
||||
if all is not None:
|
||||
params['all'] = all
|
||||
return self._result(self._post(url, params=params), True)
|
||||
|
||||
def _set_auth_headers(self, headers):
|
||||
log.debug('Looking for auth config')
|
||||
|
||||
# If we don't have any auth data so far, try reloading the config
|
||||
# file one more time in case anything showed up in there.
|
||||
if not self._auth_configs or self._auth_configs.is_empty:
|
||||
log.debug("No auth config in memory - loading from filesystem")
|
||||
self._auth_configs = auth.load_config(
|
||||
credstore_env=self.credstore_env
|
||||
)
|
||||
|
||||
# Send the full auth configuration (if any exists), since the build
|
||||
# could use any (or all) of the registries.
|
||||
if self._auth_configs:
|
||||
auth_data = self._auth_configs.get_all_credentials()
|
||||
|
||||
# See https://github.com/docker/docker-py/issues/1683
|
||||
if (auth.INDEX_URL not in auth_data and
|
||||
auth.INDEX_NAME in auth_data):
|
||||
auth_data[auth.INDEX_URL] = auth_data.get(auth.INDEX_NAME, {})
|
||||
|
||||
log.debug(
|
||||
"Sending auth config (%s)",
|
||||
', '.join(repr(k) for k in auth_data),
|
||||
)
|
||||
|
||||
if auth_data:
|
||||
headers['X-Registry-Config'] = auth.encode_header(
|
||||
auth_data
|
||||
)
|
||||
else:
|
||||
log.debug('No auth config found')
|
||||
|
||||
|
||||
def process_dockerfile(dockerfile, path):
|
||||
if not dockerfile:
|
||||
return (None, None)
|
||||
|
||||
abs_dockerfile = dockerfile
|
||||
if not os.path.isabs(dockerfile):
|
||||
abs_dockerfile = os.path.join(path, dockerfile)
|
||||
if constants.IS_WINDOWS_PLATFORM and path.startswith(
|
||||
constants.WINDOWS_LONGPATH_PREFIX):
|
||||
normpath = os.path.normpath(
|
||||
abs_dockerfile[len(constants.WINDOWS_LONGPATH_PREFIX):])
|
||||
abs_dockerfile = f'{constants.WINDOWS_LONGPATH_PREFIX}{normpath}'
|
||||
if (os.path.splitdrive(path)[0] != os.path.splitdrive(abs_dockerfile)[0] or
|
||||
os.path.relpath(abs_dockerfile, path).startswith('..')):
|
||||
# Dockerfile not in context - read data to insert into tar later
|
||||
with open(abs_dockerfile) as df:
|
||||
return (
|
||||
f'.dockerfile.{random.getrandbits(160):x}',
|
||||
df.read()
|
||||
)
|
||||
|
||||
# Dockerfile is inside the context - return path relative to context root
|
||||
if dockerfile == abs_dockerfile:
|
||||
# Only calculate relpath if necessary to avoid errors
|
||||
# on Windows client -> Linux Docker
|
||||
# see https://github.com/docker/compose/issues/5969
|
||||
dockerfile = os.path.relpath(abs_dockerfile, path)
|
||||
return (dockerfile, None)
|
||||
@@ -0,0 +1,532 @@
|
||||
import json
|
||||
import struct
|
||||
import urllib
|
||||
from functools import partial
|
||||
|
||||
import requests
|
||||
import requests.adapters
|
||||
import requests.exceptions
|
||||
|
||||
from .. import auth
|
||||
from ..constants import (
|
||||
DEFAULT_MAX_POOL_SIZE,
|
||||
DEFAULT_NUM_POOLS,
|
||||
DEFAULT_NUM_POOLS_SSH,
|
||||
DEFAULT_TIMEOUT_SECONDS,
|
||||
DEFAULT_USER_AGENT,
|
||||
IS_WINDOWS_PLATFORM,
|
||||
MINIMUM_DOCKER_API_VERSION,
|
||||
STREAM_HEADER_SIZE_BYTES,
|
||||
)
|
||||
from ..errors import (
|
||||
DockerException,
|
||||
InvalidVersion,
|
||||
TLSParameterError,
|
||||
create_api_error_from_http_exception,
|
||||
)
|
||||
from ..tls import TLSConfig
|
||||
from ..transport import UnixHTTPAdapter
|
||||
from ..utils import check_resource, config, update_headers, utils
|
||||
from ..utils.json_stream import json_stream
|
||||
from ..utils.proxy import ProxyConfig
|
||||
from ..utils.socket import consume_socket_output, demux_adaptor, frames_iter
|
||||
from .build import BuildApiMixin
|
||||
from .config import ConfigApiMixin
|
||||
from .container import ContainerApiMixin
|
||||
from .daemon import DaemonApiMixin
|
||||
from .exec_api import ExecApiMixin
|
||||
from .image import ImageApiMixin
|
||||
from .network import NetworkApiMixin
|
||||
from .plugin import PluginApiMixin
|
||||
from .secret import SecretApiMixin
|
||||
from .service import ServiceApiMixin
|
||||
from .swarm import SwarmApiMixin
|
||||
from .volume import VolumeApiMixin
|
||||
|
||||
try:
|
||||
from ..transport import NpipeHTTPAdapter
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from ..transport import SSHHTTPAdapter
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
class APIClient(
|
||||
requests.Session,
|
||||
BuildApiMixin,
|
||||
ConfigApiMixin,
|
||||
ContainerApiMixin,
|
||||
DaemonApiMixin,
|
||||
ExecApiMixin,
|
||||
ImageApiMixin,
|
||||
NetworkApiMixin,
|
||||
PluginApiMixin,
|
||||
SecretApiMixin,
|
||||
ServiceApiMixin,
|
||||
SwarmApiMixin,
|
||||
VolumeApiMixin):
|
||||
"""
|
||||
A low-level client for the Docker Engine API.
|
||||
|
||||
Example:
|
||||
|
||||
>>> import docker
|
||||
>>> client = docker.APIClient(base_url='unix://var/run/docker.sock')
|
||||
>>> client.version()
|
||||
{u'ApiVersion': u'1.33',
|
||||
u'Arch': u'amd64',
|
||||
u'BuildTime': u'2017-11-19T18:46:37.000000000+00:00',
|
||||
u'GitCommit': u'f4ffd2511c',
|
||||
u'GoVersion': u'go1.9.2',
|
||||
u'KernelVersion': u'4.14.3-1-ARCH',
|
||||
u'MinAPIVersion': u'1.12',
|
||||
u'Os': u'linux',
|
||||
u'Version': u'17.10.0-ce'}
|
||||
|
||||
Args:
|
||||
base_url (str): URL to the Docker server. For example,
|
||||
``unix:///var/run/docker.sock`` or ``tcp://127.0.0.1:1234``.
|
||||
version (str): The version of the API to use. Set to ``auto`` to
|
||||
automatically detect the server's version. Default: ``1.35``
|
||||
timeout (int): Default timeout for API calls, in seconds.
|
||||
tls (bool or :py:class:`~docker.tls.TLSConfig`): Enable TLS. Pass
|
||||
``True`` to enable it with default options, or pass a
|
||||
:py:class:`~docker.tls.TLSConfig` object to use custom
|
||||
configuration.
|
||||
user_agent (str): Set a custom user agent for requests to the server.
|
||||
credstore_env (dict): Override environment variables when calling the
|
||||
credential store process.
|
||||
use_ssh_client (bool): If set to `True`, an ssh connection is made
|
||||
via shelling out to the ssh client. Ensure the ssh client is
|
||||
installed and configured on the host.
|
||||
max_pool_size (int): The maximum number of connections
|
||||
to save in the pool.
|
||||
"""
|
||||
|
||||
__attrs__ = requests.Session.__attrs__ + ['_auth_configs',
|
||||
'_general_configs',
|
||||
'_version',
|
||||
'base_url',
|
||||
'timeout']
|
||||
|
||||
def __init__(self, base_url=None, version=None,
|
||||
timeout=DEFAULT_TIMEOUT_SECONDS, tls=False,
|
||||
user_agent=DEFAULT_USER_AGENT, num_pools=None,
|
||||
credstore_env=None, use_ssh_client=False,
|
||||
max_pool_size=DEFAULT_MAX_POOL_SIZE):
|
||||
super().__init__()
|
||||
|
||||
if tls and not base_url:
|
||||
raise TLSParameterError(
|
||||
'If using TLS, the base_url argument must be provided.'
|
||||
)
|
||||
|
||||
self.base_url = base_url
|
||||
self.timeout = timeout
|
||||
self.headers['User-Agent'] = user_agent
|
||||
|
||||
self._general_configs = config.load_general_config()
|
||||
|
||||
proxy_config = self._general_configs.get('proxies', {})
|
||||
try:
|
||||
proxies = proxy_config[base_url]
|
||||
except KeyError:
|
||||
proxies = proxy_config.get('default', {})
|
||||
|
||||
self._proxy_configs = ProxyConfig.from_dict(proxies)
|
||||
|
||||
self._auth_configs = auth.load_config(
|
||||
config_dict=self._general_configs, credstore_env=credstore_env,
|
||||
)
|
||||
self.credstore_env = credstore_env
|
||||
|
||||
base_url = utils.parse_host(
|
||||
base_url, IS_WINDOWS_PLATFORM, tls=bool(tls)
|
||||
)
|
||||
# SSH has a different default for num_pools to all other adapters
|
||||
num_pools = num_pools or DEFAULT_NUM_POOLS_SSH if \
|
||||
base_url.startswith('ssh://') else DEFAULT_NUM_POOLS
|
||||
|
||||
if base_url.startswith('http+unix://'):
|
||||
self._custom_adapter = UnixHTTPAdapter(
|
||||
base_url, timeout, pool_connections=num_pools,
|
||||
max_pool_size=max_pool_size
|
||||
)
|
||||
self.mount('http+docker://', self._custom_adapter)
|
||||
self._unmount('http://', 'https://')
|
||||
# host part of URL should be unused, but is resolved by requests
|
||||
# module in proxy_bypass_macosx_sysconf()
|
||||
self.base_url = 'http+docker://localhost'
|
||||
elif base_url.startswith('npipe://'):
|
||||
if not IS_WINDOWS_PLATFORM:
|
||||
raise DockerException(
|
||||
'The npipe:// protocol is only supported on Windows'
|
||||
)
|
||||
try:
|
||||
self._custom_adapter = NpipeHTTPAdapter(
|
||||
base_url, timeout, pool_connections=num_pools,
|
||||
max_pool_size=max_pool_size
|
||||
)
|
||||
except NameError as err:
|
||||
raise DockerException(
|
||||
'Install pypiwin32 package to enable npipe:// support'
|
||||
) from err
|
||||
self.mount('http+docker://', self._custom_adapter)
|
||||
self.base_url = 'http+docker://localnpipe'
|
||||
elif base_url.startswith('ssh://'):
|
||||
try:
|
||||
self._custom_adapter = SSHHTTPAdapter(
|
||||
base_url, timeout, pool_connections=num_pools,
|
||||
max_pool_size=max_pool_size, shell_out=use_ssh_client
|
||||
)
|
||||
except NameError as err:
|
||||
raise DockerException(
|
||||
'Install paramiko package to enable ssh:// support'
|
||||
) from err
|
||||
self.mount('http+docker://ssh', self._custom_adapter)
|
||||
self._unmount('http://', 'https://')
|
||||
self.base_url = 'http+docker://ssh'
|
||||
else:
|
||||
# Use SSLAdapter for the ability to specify SSL version
|
||||
if isinstance(tls, TLSConfig):
|
||||
tls.configure_client(self)
|
||||
elif tls:
|
||||
self._custom_adapter = requests.adapters.HTTPAdapter(
|
||||
pool_connections=num_pools)
|
||||
self.mount('https://', self._custom_adapter)
|
||||
self.base_url = base_url
|
||||
|
||||
# version detection needs to be after unix adapter mounting
|
||||
if version is None or (isinstance(
|
||||
version,
|
||||
str
|
||||
) and version.lower() == 'auto'):
|
||||
self._version = self._retrieve_server_version()
|
||||
else:
|
||||
self._version = version
|
||||
if not isinstance(self._version, str):
|
||||
raise DockerException(
|
||||
'Version parameter must be a string or None. '
|
||||
f'Found {type(version).__name__}'
|
||||
)
|
||||
if utils.version_lt(self._version, MINIMUM_DOCKER_API_VERSION):
|
||||
raise InvalidVersion(
|
||||
f'API versions below {MINIMUM_DOCKER_API_VERSION} are '
|
||||
f'no longer supported by this library.'
|
||||
)
|
||||
|
||||
def _retrieve_server_version(self):
|
||||
try:
|
||||
return self.version(api_version=False)["ApiVersion"]
|
||||
except KeyError as ke:
|
||||
raise DockerException(
|
||||
'Invalid response from docker daemon: key "ApiVersion"'
|
||||
' is missing.'
|
||||
) from ke
|
||||
except Exception as e:
|
||||
raise DockerException(
|
||||
f'Error while fetching server API version: {e}'
|
||||
) from e
|
||||
|
||||
def _set_request_timeout(self, kwargs):
|
||||
"""Prepare the kwargs for an HTTP request by inserting the timeout
|
||||
parameter, if not already present."""
|
||||
kwargs.setdefault('timeout', self.timeout)
|
||||
return kwargs
|
||||
|
||||
@update_headers
|
||||
def _post(self, url, **kwargs):
|
||||
return self.post(url, **self._set_request_timeout(kwargs))
|
||||
|
||||
@update_headers
|
||||
def _get(self, url, **kwargs):
|
||||
return self.get(url, **self._set_request_timeout(kwargs))
|
||||
|
||||
@update_headers
|
||||
def _put(self, url, **kwargs):
|
||||
return self.put(url, **self._set_request_timeout(kwargs))
|
||||
|
||||
@update_headers
|
||||
def _delete(self, url, **kwargs):
|
||||
return self.delete(url, **self._set_request_timeout(kwargs))
|
||||
|
||||
def _url(self, pathfmt, *args, **kwargs):
|
||||
for arg in args:
|
||||
if not isinstance(arg, str):
|
||||
raise ValueError(
|
||||
f'Expected a string but found {arg} ({type(arg)}) instead'
|
||||
)
|
||||
|
||||
quote_f = partial(urllib.parse.quote, safe="/:")
|
||||
args = map(quote_f, args)
|
||||
|
||||
formatted_path = pathfmt.format(*args)
|
||||
if kwargs.get('versioned_api', True):
|
||||
return f'{self.base_url}/v{self._version}{formatted_path}'
|
||||
else:
|
||||
return f'{self.base_url}{formatted_path}'
|
||||
|
||||
def _raise_for_status(self, response):
|
||||
"""Raises stored :class:`APIError`, if one occurred."""
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except requests.exceptions.HTTPError as e:
|
||||
raise create_api_error_from_http_exception(e) from e
|
||||
|
||||
def _result(self, response, json=False, binary=False):
|
||||
assert not (json and binary)
|
||||
self._raise_for_status(response)
|
||||
|
||||
if json:
|
||||
return response.json()
|
||||
if binary:
|
||||
return response.content
|
||||
return response.text
|
||||
|
||||
def _post_json(self, url, data, **kwargs):
|
||||
# Go <1.1 can't unserialize null to a string
|
||||
# so we do this disgusting thing here.
|
||||
data2 = {}
|
||||
if data is not None and isinstance(data, dict):
|
||||
for k, v in iter(data.items()):
|
||||
if v is not None:
|
||||
data2[k] = v
|
||||
elif data is not None:
|
||||
data2 = data
|
||||
|
||||
if 'headers' not in kwargs:
|
||||
kwargs['headers'] = {}
|
||||
kwargs['headers']['Content-Type'] = 'application/json'
|
||||
return self._post(url, data=json.dumps(data2), **kwargs)
|
||||
|
||||
def _attach_params(self, override=None):
|
||||
return override or {
|
||||
'stdout': 1,
|
||||
'stderr': 1,
|
||||
'stream': 1
|
||||
}
|
||||
|
||||
@check_resource('container')
|
||||
def _attach_websocket(self, container, params=None):
|
||||
url = self._url("/containers/{0}/attach/ws", container)
|
||||
req = requests.Request("POST", url, params=self._attach_params(params))
|
||||
full_url = req.prepare().url
|
||||
full_url = full_url.replace("http://", "ws://", 1)
|
||||
full_url = full_url.replace("https://", "wss://", 1)
|
||||
return self._create_websocket_connection(full_url)
|
||||
|
||||
def _create_websocket_connection(self, url):
|
||||
try:
|
||||
import websocket
|
||||
return websocket.create_connection(url)
|
||||
except ImportError as ie:
|
||||
raise DockerException(
|
||||
'The `websocket-client` library is required '
|
||||
'for using websocket connections. '
|
||||
'You can install the `docker` library '
|
||||
'with the [websocket] extra to install it.'
|
||||
) from ie
|
||||
|
||||
def _get_raw_response_socket(self, response):
|
||||
self._raise_for_status(response)
|
||||
if self.base_url == "http+docker://localnpipe":
|
||||
sock = response.raw._fp.fp.raw.sock
|
||||
elif self.base_url.startswith('http+docker://ssh'):
|
||||
sock = response.raw._fp.fp.channel
|
||||
else:
|
||||
sock = response.raw._fp.fp.raw
|
||||
if self.base_url.startswith("https://"):
|
||||
sock = sock._sock
|
||||
try:
|
||||
# Keep a reference to the response to stop it being garbage
|
||||
# collected. If the response is garbage collected, it will
|
||||
# close TLS sockets.
|
||||
sock._response = response
|
||||
except AttributeError:
|
||||
# UNIX sockets can't have attributes set on them, but that's
|
||||
# fine because we won't be doing TLS over them
|
||||
pass
|
||||
|
||||
return sock
|
||||
|
||||
def _stream_helper(self, response, decode=False):
|
||||
"""Generator for data coming from a chunked-encoded HTTP response."""
|
||||
|
||||
if response.raw._fp.chunked:
|
||||
if decode:
|
||||
yield from json_stream(self._stream_helper(response, False))
|
||||
else:
|
||||
reader = response.raw
|
||||
while not reader.closed:
|
||||
# this read call will block until we get a chunk
|
||||
data = reader.read(1)
|
||||
if not data:
|
||||
break
|
||||
if reader._fp.chunk_left:
|
||||
data += reader.read(reader._fp.chunk_left)
|
||||
yield data
|
||||
else:
|
||||
# Response isn't chunked, meaning we probably
|
||||
# encountered an error immediately
|
||||
yield self._result(response, json=decode)
|
||||
|
||||
def _multiplexed_buffer_helper(self, response):
|
||||
"""A generator of multiplexed data blocks read from a buffered
|
||||
response."""
|
||||
buf = self._result(response, binary=True)
|
||||
buf_length = len(buf)
|
||||
walker = 0
|
||||
while True:
|
||||
if buf_length - walker < STREAM_HEADER_SIZE_BYTES:
|
||||
break
|
||||
header = buf[walker:walker + STREAM_HEADER_SIZE_BYTES]
|
||||
_, length = struct.unpack_from('>BxxxL', header)
|
||||
start = walker + STREAM_HEADER_SIZE_BYTES
|
||||
end = start + length
|
||||
walker = end
|
||||
yield buf[start:end]
|
||||
|
||||
def _multiplexed_response_stream_helper(self, response):
|
||||
"""A generator of multiplexed data blocks coming from a response
|
||||
stream."""
|
||||
|
||||
# Disable timeout on the underlying socket to prevent
|
||||
# Read timed out(s) for long running processes
|
||||
socket = self._get_raw_response_socket(response)
|
||||
self._disable_socket_timeout(socket)
|
||||
|
||||
while True:
|
||||
header = response.raw.read(STREAM_HEADER_SIZE_BYTES)
|
||||
if not header:
|
||||
break
|
||||
_, length = struct.unpack('>BxxxL', header)
|
||||
if not length:
|
||||
continue
|
||||
data = response.raw.read(length)
|
||||
if not data:
|
||||
break
|
||||
yield data
|
||||
|
||||
def _stream_raw_result(self, response, chunk_size=1, decode=True):
|
||||
''' Stream result for TTY-enabled container and raw binary data'''
|
||||
self._raise_for_status(response)
|
||||
|
||||
# Disable timeout on the underlying socket to prevent
|
||||
# Read timed out(s) for long running processes
|
||||
socket = self._get_raw_response_socket(response)
|
||||
self._disable_socket_timeout(socket)
|
||||
|
||||
yield from response.iter_content(chunk_size, decode)
|
||||
|
||||
def _read_from_socket(self, response, stream, tty=True, demux=False):
|
||||
"""Consume all data from the socket, close the response and return the
|
||||
data. If stream=True, then a generator is returned instead and the
|
||||
caller is responsible for closing the response.
|
||||
"""
|
||||
socket = self._get_raw_response_socket(response)
|
||||
|
||||
gen = frames_iter(socket, tty)
|
||||
|
||||
if demux:
|
||||
# The generator will output tuples (stdout, stderr)
|
||||
gen = (demux_adaptor(*frame) for frame in gen)
|
||||
else:
|
||||
# The generator will output strings
|
||||
gen = (data for (_, data) in gen)
|
||||
|
||||
if stream:
|
||||
return gen
|
||||
else:
|
||||
try:
|
||||
# Wait for all frames, concatenate them, and return the result
|
||||
return consume_socket_output(gen, demux=demux)
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
def _disable_socket_timeout(self, socket):
|
||||
""" Depending on the combination of python version and whether we're
|
||||
connecting over http or https, we might need to access _sock, which
|
||||
may or may not exist; or we may need to just settimeout on socket
|
||||
itself, which also may or may not have settimeout on it. To avoid
|
||||
missing the correct one, we try both.
|
||||
|
||||
We also do not want to set the timeout if it is already disabled, as
|
||||
you run the risk of changing a socket that was non-blocking to
|
||||
blocking, for example when using gevent.
|
||||
"""
|
||||
sockets = [socket, getattr(socket, '_sock', None)]
|
||||
|
||||
for s in sockets:
|
||||
if not hasattr(s, 'settimeout'):
|
||||
continue
|
||||
|
||||
timeout = -1
|
||||
|
||||
if hasattr(s, 'gettimeout'):
|
||||
timeout = s.gettimeout()
|
||||
|
||||
# Don't change the timeout if it is already disabled.
|
||||
if timeout is None or timeout == 0.0:
|
||||
continue
|
||||
|
||||
s.settimeout(None)
|
||||
|
||||
@check_resource('container')
|
||||
def _check_is_tty(self, container):
|
||||
cont = self.inspect_container(container)
|
||||
return cont['Config']['Tty']
|
||||
|
||||
def _get_result(self, container, stream, res):
|
||||
return self._get_result_tty(stream, res, self._check_is_tty(container))
|
||||
|
||||
def _get_result_tty(self, stream, res, is_tty):
|
||||
# We should also use raw streaming (without keep-alives)
|
||||
# if we're dealing with a tty-enabled container.
|
||||
if is_tty:
|
||||
return self._stream_raw_result(res) if stream else \
|
||||
self._result(res, binary=True)
|
||||
|
||||
self._raise_for_status(res)
|
||||
sep = b''
|
||||
if stream:
|
||||
return self._multiplexed_response_stream_helper(res)
|
||||
else:
|
||||
return sep.join(
|
||||
list(self._multiplexed_buffer_helper(res))
|
||||
)
|
||||
|
||||
def _unmount(self, *args):
|
||||
for proto in args:
|
||||
self.adapters.pop(proto)
|
||||
|
||||
def get_adapter(self, url):
|
||||
try:
|
||||
return super().get_adapter(url)
|
||||
except requests.exceptions.InvalidSchema as e:
|
||||
if self._custom_adapter:
|
||||
return self._custom_adapter
|
||||
else:
|
||||
raise e
|
||||
|
||||
@property
|
||||
def api_version(self):
|
||||
return self._version
|
||||
|
||||
def reload_config(self, dockercfg_path=None):
|
||||
"""
|
||||
Force a reload of the auth configuration
|
||||
|
||||
Args:
|
||||
dockercfg_path (str): Use a custom path for the Docker config file
|
||||
(default ``$HOME/.docker/config.json`` if present,
|
||||
otherwise ``$HOME/.dockercfg``)
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
self._auth_configs = auth.load_config(
|
||||
dockercfg_path, credstore_env=self.credstore_env
|
||||
)
|
||||
@@ -0,0 +1,92 @@
|
||||
import base64
|
||||
|
||||
from .. import utils
|
||||
|
||||
|
||||
class ConfigApiMixin:
|
||||
@utils.minimum_version('1.30')
|
||||
def create_config(self, name, data, labels=None, templating=None):
|
||||
"""
|
||||
Create a config
|
||||
|
||||
Args:
|
||||
name (string): Name of the config
|
||||
data (bytes): Config data to be stored
|
||||
labels (dict): A mapping of labels to assign to the config
|
||||
templating (dict): dictionary containing the name of the
|
||||
templating driver to be used expressed as
|
||||
{ name: <templating_driver_name>}
|
||||
|
||||
Returns (dict): ID of the newly created config
|
||||
"""
|
||||
if not isinstance(data, bytes):
|
||||
data = data.encode('utf-8')
|
||||
|
||||
data = base64.b64encode(data)
|
||||
data = data.decode('ascii')
|
||||
body = {
|
||||
'Data': data,
|
||||
'Name': name,
|
||||
'Labels': labels,
|
||||
'Templating': templating
|
||||
}
|
||||
|
||||
url = self._url('/configs/create')
|
||||
return self._result(
|
||||
self._post_json(url, data=body), True
|
||||
)
|
||||
|
||||
@utils.minimum_version('1.30')
|
||||
@utils.check_resource('id')
|
||||
def inspect_config(self, id):
|
||||
"""
|
||||
Retrieve config metadata
|
||||
|
||||
Args:
|
||||
id (string): Full ID of the config to inspect
|
||||
|
||||
Returns (dict): A dictionary of metadata
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.NotFound`
|
||||
if no config with that ID exists
|
||||
"""
|
||||
url = self._url('/configs/{0}', id)
|
||||
return self._result(self._get(url), True)
|
||||
|
||||
@utils.minimum_version('1.30')
|
||||
@utils.check_resource('id')
|
||||
def remove_config(self, id):
|
||||
"""
|
||||
Remove a config
|
||||
|
||||
Args:
|
||||
id (string): Full ID of the config to remove
|
||||
|
||||
Returns (boolean): True if successful
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.NotFound`
|
||||
if no config with that ID exists
|
||||
"""
|
||||
url = self._url('/configs/{0}', id)
|
||||
res = self._delete(url)
|
||||
self._raise_for_status(res)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.30')
|
||||
def configs(self, filters=None):
|
||||
"""
|
||||
List configs
|
||||
|
||||
Args:
|
||||
filters (dict): A map of filters to process on the configs
|
||||
list. Available filters: ``names``
|
||||
|
||||
Returns (list): A list of configs
|
||||
"""
|
||||
url = self._url('/configs')
|
||||
params = {}
|
||||
if filters:
|
||||
params['filters'] = utils.convert_filters(filters)
|
||||
return self._result(self._get(url, params=params), True)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,181 @@
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
from .. import auth, types, utils
|
||||
|
||||
|
||||
class DaemonApiMixin:
|
||||
@utils.minimum_version('1.25')
|
||||
def df(self):
|
||||
"""
|
||||
Get data usage information.
|
||||
|
||||
Returns:
|
||||
(dict): A dictionary representing different resource categories
|
||||
and their respective data usage.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
url = self._url('/system/df')
|
||||
return self._result(self._get(url), True)
|
||||
|
||||
def events(self, since=None, until=None, filters=None, decode=None):
|
||||
"""
|
||||
Get real-time events from the server. Similar to the ``docker events``
|
||||
command.
|
||||
|
||||
Args:
|
||||
since (UTC datetime or int): Get events from this point
|
||||
until (UTC datetime or int): Get events until this point
|
||||
filters (dict): Filter the events by event time, container or image
|
||||
decode (bool): If set to true, stream will be decoded into dicts on
|
||||
the fly. False by default.
|
||||
|
||||
Returns:
|
||||
A :py:class:`docker.types.daemon.CancellableStream` generator
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> for event in client.events(decode=True)
|
||||
... print(event)
|
||||
{u'from': u'image/with:tag',
|
||||
u'id': u'container-id',
|
||||
u'status': u'start',
|
||||
u'time': 1423339459}
|
||||
...
|
||||
|
||||
or
|
||||
|
||||
>>> events = client.events()
|
||||
>>> for event in events:
|
||||
... print(event)
|
||||
>>> # and cancel from another thread
|
||||
>>> events.close()
|
||||
"""
|
||||
|
||||
if isinstance(since, datetime):
|
||||
since = utils.datetime_to_timestamp(since)
|
||||
|
||||
if isinstance(until, datetime):
|
||||
until = utils.datetime_to_timestamp(until)
|
||||
|
||||
if filters:
|
||||
filters = utils.convert_filters(filters)
|
||||
|
||||
params = {
|
||||
'since': since,
|
||||
'until': until,
|
||||
'filters': filters
|
||||
}
|
||||
url = self._url('/events')
|
||||
|
||||
response = self._get(url, params=params, stream=True, timeout=None)
|
||||
stream = self._stream_helper(response, decode=decode)
|
||||
|
||||
return types.CancellableStream(stream, response)
|
||||
|
||||
def info(self):
|
||||
"""
|
||||
Display system-wide information. Identical to the ``docker info``
|
||||
command.
|
||||
|
||||
Returns:
|
||||
(dict): The info as a dict
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self._result(self._get(self._url("/info")), True)
|
||||
|
||||
def login(self, username, password=None, email=None, registry=None,
|
||||
reauth=False, dockercfg_path=None):
|
||||
"""
|
||||
Authenticate with a registry. Similar to the ``docker login`` command.
|
||||
|
||||
Args:
|
||||
username (str): The registry username
|
||||
password (str): The plaintext password
|
||||
email (str): The email for the registry account
|
||||
registry (str): URL to the registry. E.g.
|
||||
``https://index.docker.io/v1/``
|
||||
reauth (bool): Whether or not to refresh existing authentication on
|
||||
the Docker server.
|
||||
dockercfg_path (str): Use a custom path for the Docker config file
|
||||
(default ``$HOME/.docker/config.json`` if present,
|
||||
otherwise ``$HOME/.dockercfg``)
|
||||
|
||||
Returns:
|
||||
(dict): The response from the login request
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
|
||||
# If we don't have any auth data so far, try reloading the config file
|
||||
# one more time in case anything showed up in there.
|
||||
# If dockercfg_path is passed check to see if the config file exists,
|
||||
# if so load that config.
|
||||
if dockercfg_path and os.path.exists(dockercfg_path):
|
||||
self._auth_configs = auth.load_config(
|
||||
dockercfg_path, credstore_env=self.credstore_env
|
||||
)
|
||||
elif not self._auth_configs or self._auth_configs.is_empty:
|
||||
self._auth_configs = auth.load_config(
|
||||
credstore_env=self.credstore_env
|
||||
)
|
||||
|
||||
authcfg = self._auth_configs.resolve_authconfig(registry)
|
||||
# If we found an existing auth config for this registry and username
|
||||
# combination, we can return it immediately unless reauth is requested.
|
||||
if authcfg and authcfg.get('username', None) == username \
|
||||
and not reauth:
|
||||
return authcfg
|
||||
|
||||
req_data = {
|
||||
'username': username,
|
||||
'password': password,
|
||||
'email': email,
|
||||
'serveraddress': registry,
|
||||
}
|
||||
|
||||
response = self._post_json(self._url('/auth'), data=req_data)
|
||||
if response.status_code == 200:
|
||||
self._auth_configs.add_auth(registry or auth.INDEX_NAME, req_data)
|
||||
return self._result(response, json=True)
|
||||
|
||||
def ping(self):
|
||||
"""
|
||||
Checks the server is responsive. An exception will be raised if it
|
||||
isn't responding.
|
||||
|
||||
Returns:
|
||||
(bool) The response from the server.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self._result(self._get(self._url('/_ping'))) == 'OK'
|
||||
|
||||
def version(self, api_version=True):
|
||||
"""
|
||||
Returns version information from the server. Similar to the ``docker
|
||||
version`` command.
|
||||
|
||||
Returns:
|
||||
(dict): The server version information
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
url = self._url("/version", versioned_api=api_version)
|
||||
return self._result(self._get(url), json=True)
|
||||
@@ -0,0 +1,176 @@
|
||||
from .. import errors, utils
|
||||
from ..types import CancellableStream
|
||||
|
||||
|
||||
class ExecApiMixin:
|
||||
@utils.check_resource('container')
|
||||
def exec_create(self, container, cmd, stdout=True, stderr=True,
|
||||
stdin=False, tty=False, privileged=False, user='',
|
||||
environment=None, workdir=None, detach_keys=None):
|
||||
"""
|
||||
Sets up an exec instance in a running container.
|
||||
|
||||
Args:
|
||||
container (str): Target container where exec instance will be
|
||||
created
|
||||
cmd (str or list): Command to be executed
|
||||
stdout (bool): Attach to stdout. Default: ``True``
|
||||
stderr (bool): Attach to stderr. Default: ``True``
|
||||
stdin (bool): Attach to stdin. Default: ``False``
|
||||
tty (bool): Allocate a pseudo-TTY. Default: False
|
||||
privileged (bool): Run as privileged.
|
||||
user (str): User to execute command as. Default: root
|
||||
environment (dict or list): A dictionary or a list of strings in
|
||||
the following format ``["PASSWORD=xxx"]`` or
|
||||
``{"PASSWORD": "xxx"}``.
|
||||
workdir (str): Path to working directory for this exec session
|
||||
detach_keys (str): Override the key sequence for detaching
|
||||
a container. Format is a single character `[a-Z]`
|
||||
or `ctrl-<value>` where `<value>` is one of:
|
||||
`a-z`, `@`, `^`, `[`, `,` or `_`.
|
||||
~/.docker/config.json is used by default.
|
||||
|
||||
Returns:
|
||||
(dict): A dictionary with an exec ``Id`` key.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
|
||||
if environment is not None and utils.version_lt(self._version, '1.25'):
|
||||
raise errors.InvalidVersion(
|
||||
'Setting environment for exec is not supported in API < 1.25'
|
||||
)
|
||||
|
||||
if isinstance(cmd, str):
|
||||
cmd = utils.split_command(cmd)
|
||||
|
||||
if isinstance(environment, dict):
|
||||
environment = utils.utils.format_environment(environment)
|
||||
|
||||
data = {
|
||||
'Container': container,
|
||||
'User': user,
|
||||
'Privileged': privileged,
|
||||
'Tty': tty,
|
||||
'AttachStdin': stdin,
|
||||
'AttachStdout': stdout,
|
||||
'AttachStderr': stderr,
|
||||
'Cmd': cmd,
|
||||
'Env': environment,
|
||||
}
|
||||
|
||||
if workdir is not None:
|
||||
if utils.version_lt(self._version, '1.35'):
|
||||
raise errors.InvalidVersion(
|
||||
'workdir is not supported for API version < 1.35'
|
||||
)
|
||||
data['WorkingDir'] = workdir
|
||||
|
||||
if detach_keys:
|
||||
data['detachKeys'] = detach_keys
|
||||
elif 'detachKeys' in self._general_configs:
|
||||
data['detachKeys'] = self._general_configs['detachKeys']
|
||||
|
||||
url = self._url('/containers/{0}/exec', container)
|
||||
res = self._post_json(url, data=data)
|
||||
return self._result(res, True)
|
||||
|
||||
def exec_inspect(self, exec_id):
|
||||
"""
|
||||
Return low-level information about an exec command.
|
||||
|
||||
Args:
|
||||
exec_id (str): ID of the exec instance
|
||||
|
||||
Returns:
|
||||
(dict): Dictionary of values returned by the endpoint.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
if isinstance(exec_id, dict):
|
||||
exec_id = exec_id.get('Id')
|
||||
res = self._get(self._url("/exec/{0}/json", exec_id))
|
||||
return self._result(res, True)
|
||||
|
||||
def exec_resize(self, exec_id, height=None, width=None):
|
||||
"""
|
||||
Resize the tty session used by the specified exec command.
|
||||
|
||||
Args:
|
||||
exec_id (str): ID of the exec instance
|
||||
height (int): Height of tty session
|
||||
width (int): Width of tty session
|
||||
"""
|
||||
|
||||
if isinstance(exec_id, dict):
|
||||
exec_id = exec_id.get('Id')
|
||||
|
||||
params = {'h': height, 'w': width}
|
||||
url = self._url("/exec/{0}/resize", exec_id)
|
||||
res = self._post(url, params=params)
|
||||
self._raise_for_status(res)
|
||||
|
||||
@utils.check_resource('exec_id')
|
||||
def exec_start(self, exec_id, detach=False, tty=False, stream=False,
|
||||
socket=False, demux=False):
|
||||
"""
|
||||
Start a previously set up exec instance.
|
||||
|
||||
Args:
|
||||
exec_id (str): ID of the exec instance
|
||||
detach (bool): If true, detach from the exec command.
|
||||
Default: False
|
||||
tty (bool): Allocate a pseudo-TTY. Default: False
|
||||
stream (bool): Return response data progressively as an iterator
|
||||
of strings, rather than a single string.
|
||||
socket (bool): Return the connection socket to allow custom
|
||||
read/write operations. Must be closed by the caller when done.
|
||||
demux (bool): Return stdout and stderr separately
|
||||
|
||||
Returns:
|
||||
|
||||
(generator or str or tuple): If ``stream=True``, a generator
|
||||
yielding response chunks. If ``socket=True``, a socket object for
|
||||
the connection. A string containing response data otherwise. If
|
||||
``demux=True``, a tuple with two elements of type byte: stdout and
|
||||
stderr.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
# we want opened socket if socket == True
|
||||
|
||||
data = {
|
||||
'Tty': tty,
|
||||
'Detach': detach
|
||||
}
|
||||
|
||||
headers = {} if detach else {
|
||||
'Connection': 'Upgrade',
|
||||
'Upgrade': 'tcp'
|
||||
}
|
||||
|
||||
res = self._post_json(
|
||||
self._url('/exec/{0}/start', exec_id),
|
||||
headers=headers,
|
||||
data=data,
|
||||
stream=True
|
||||
)
|
||||
if detach:
|
||||
try:
|
||||
return self._result(res)
|
||||
finally:
|
||||
res.close()
|
||||
if socket:
|
||||
return self._get_raw_response_socket(res)
|
||||
|
||||
output = self._read_from_socket(res, stream, tty=tty, demux=demux)
|
||||
if stream:
|
||||
return CancellableStream(output, res)
|
||||
else:
|
||||
return output
|
||||
@@ -0,0 +1,601 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from .. import auth, errors, utils
|
||||
from ..constants import DEFAULT_DATA_CHUNK_SIZE
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ImageApiMixin:
|
||||
|
||||
@utils.check_resource('image')
|
||||
def get_image(self, image, chunk_size=DEFAULT_DATA_CHUNK_SIZE):
|
||||
"""
|
||||
Get a tarball of an image. Similar to the ``docker save`` command.
|
||||
|
||||
Args:
|
||||
image (str): Image name to get
|
||||
chunk_size (int): The number of bytes returned by each iteration
|
||||
of the generator. If ``None``, data will be streamed as it is
|
||||
received. Default: 2 MB
|
||||
|
||||
Returns:
|
||||
(generator): A stream of raw archive data.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> image = client.api.get_image("busybox:latest")
|
||||
>>> f = open('/tmp/busybox-latest.tar', 'wb')
|
||||
>>> for chunk in image:
|
||||
>>> f.write(chunk)
|
||||
>>> f.close()
|
||||
"""
|
||||
res = self._get(self._url("/images/{0}/get", image), stream=True)
|
||||
return self._stream_raw_result(res, chunk_size, False)
|
||||
|
||||
@utils.check_resource('image')
|
||||
def history(self, image):
|
||||
"""
|
||||
Show the history of an image.
|
||||
|
||||
Args:
|
||||
image (str): The image to show history for
|
||||
|
||||
Returns:
|
||||
(list): The history of the image
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
res = self._get(self._url("/images/{0}/history", image))
|
||||
return self._result(res, True)
|
||||
|
||||
def images(self, name=None, quiet=False, all=False, filters=None):
|
||||
"""
|
||||
List images. Similar to the ``docker images`` command.
|
||||
|
||||
Args:
|
||||
name (str): Only show images belonging to the repository ``name``
|
||||
quiet (bool): Only return numeric IDs as a list.
|
||||
all (bool): Show intermediate image layers. By default, these are
|
||||
filtered out.
|
||||
filters (dict): Filters to be processed on the image list.
|
||||
Available filters:
|
||||
- ``dangling`` (bool)
|
||||
- `label` (str|list): format either ``"key"``, ``"key=value"``
|
||||
or a list of such.
|
||||
|
||||
Returns:
|
||||
(dict or list): A list if ``quiet=True``, otherwise a dict.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
params = {
|
||||
'only_ids': 1 if quiet else 0,
|
||||
'all': 1 if all else 0,
|
||||
}
|
||||
if name:
|
||||
if utils.version_lt(self._version, '1.25'):
|
||||
# only use "filter" on API 1.24 and under, as it is deprecated
|
||||
params['filter'] = name
|
||||
else:
|
||||
if filters:
|
||||
filters['reference'] = name
|
||||
else:
|
||||
filters = {'reference': name}
|
||||
if filters:
|
||||
params['filters'] = utils.convert_filters(filters)
|
||||
res = self._result(self._get(self._url("/images/json"), params=params),
|
||||
True)
|
||||
if quiet:
|
||||
return [x['Id'] for x in res]
|
||||
return res
|
||||
|
||||
def import_image(self, src=None, repository=None, tag=None, image=None,
|
||||
changes=None, stream_src=False):
|
||||
"""
|
||||
Import an image. Similar to the ``docker import`` command.
|
||||
|
||||
If ``src`` is a string or unicode string, it will first be treated as a
|
||||
path to a tarball on the local system. If there is an error reading
|
||||
from that file, ``src`` will be treated as a URL instead to fetch the
|
||||
image from. You can also pass an open file handle as ``src``, in which
|
||||
case the data will be read from that file.
|
||||
|
||||
If ``src`` is unset but ``image`` is set, the ``image`` parameter will
|
||||
be taken as the name of an existing image to import from.
|
||||
|
||||
Args:
|
||||
src (str or file): Path to tarfile, URL, or file-like object
|
||||
repository (str): The repository to create
|
||||
tag (str): The tag to apply
|
||||
image (str): Use another image like the ``FROM`` Dockerfile
|
||||
parameter
|
||||
"""
|
||||
if not (src or image):
|
||||
raise errors.DockerException(
|
||||
'Must specify src or image to import from'
|
||||
)
|
||||
u = self._url('/images/create')
|
||||
|
||||
params = _import_image_params(
|
||||
repository, tag, image,
|
||||
src=(src if isinstance(src, str) else None),
|
||||
changes=changes
|
||||
)
|
||||
headers = {'Content-Type': 'application/tar'}
|
||||
|
||||
if image or params.get('fromSrc') != '-': # from image or URL
|
||||
return self._result(
|
||||
self._post(u, data=None, params=params)
|
||||
)
|
||||
elif isinstance(src, str): # from file path
|
||||
with open(src, 'rb') as f:
|
||||
return self._result(
|
||||
self._post(
|
||||
u, data=f, params=params, headers=headers, timeout=None
|
||||
)
|
||||
)
|
||||
else: # from raw data
|
||||
if stream_src:
|
||||
headers['Transfer-Encoding'] = 'chunked'
|
||||
return self._result(
|
||||
self._post(u, data=src, params=params, headers=headers)
|
||||
)
|
||||
|
||||
def import_image_from_data(self, data, repository=None, tag=None,
|
||||
changes=None):
|
||||
"""
|
||||
Like :py:meth:`~docker.api.image.ImageApiMixin.import_image`, but
|
||||
allows importing in-memory bytes data.
|
||||
|
||||
Args:
|
||||
data (bytes collection): Bytes collection containing valid tar data
|
||||
repository (str): The repository to create
|
||||
tag (str): The tag to apply
|
||||
"""
|
||||
|
||||
u = self._url('/images/create')
|
||||
params = _import_image_params(
|
||||
repository, tag, src='-', changes=changes
|
||||
)
|
||||
headers = {'Content-Type': 'application/tar'}
|
||||
return self._result(
|
||||
self._post(
|
||||
u, data=data, params=params, headers=headers, timeout=None
|
||||
)
|
||||
)
|
||||
|
||||
def import_image_from_file(self, filename, repository=None, tag=None,
|
||||
changes=None):
|
||||
"""
|
||||
Like :py:meth:`~docker.api.image.ImageApiMixin.import_image`, but only
|
||||
supports importing from a tar file on disk.
|
||||
|
||||
Args:
|
||||
filename (str): Full path to a tar file.
|
||||
repository (str): The repository to create
|
||||
tag (str): The tag to apply
|
||||
|
||||
Raises:
|
||||
IOError: File does not exist.
|
||||
"""
|
||||
|
||||
return self.import_image(
|
||||
src=filename, repository=repository, tag=tag, changes=changes
|
||||
)
|
||||
|
||||
def import_image_from_stream(self, stream, repository=None, tag=None,
|
||||
changes=None):
|
||||
return self.import_image(
|
||||
src=stream, stream_src=True, repository=repository, tag=tag,
|
||||
changes=changes
|
||||
)
|
||||
|
||||
def import_image_from_url(self, url, repository=None, tag=None,
|
||||
changes=None):
|
||||
"""
|
||||
Like :py:meth:`~docker.api.image.ImageApiMixin.import_image`, but only
|
||||
supports importing from a URL.
|
||||
|
||||
Args:
|
||||
url (str): A URL pointing to a tar file.
|
||||
repository (str): The repository to create
|
||||
tag (str): The tag to apply
|
||||
"""
|
||||
return self.import_image(
|
||||
src=url, repository=repository, tag=tag, changes=changes
|
||||
)
|
||||
|
||||
def import_image_from_image(self, image, repository=None, tag=None,
|
||||
changes=None):
|
||||
"""
|
||||
Like :py:meth:`~docker.api.image.ImageApiMixin.import_image`, but only
|
||||
supports importing from another image, like the ``FROM`` Dockerfile
|
||||
parameter.
|
||||
|
||||
Args:
|
||||
image (str): Image name to import from
|
||||
repository (str): The repository to create
|
||||
tag (str): The tag to apply
|
||||
"""
|
||||
return self.import_image(
|
||||
image=image, repository=repository, tag=tag, changes=changes
|
||||
)
|
||||
|
||||
@utils.check_resource('image')
|
||||
def inspect_image(self, image):
|
||||
"""
|
||||
Get detailed information about an image. Similar to the ``docker
|
||||
inspect`` command, but only for images.
|
||||
|
||||
Args:
|
||||
image (str): The image to inspect
|
||||
|
||||
Returns:
|
||||
(dict): Similar to the output of ``docker inspect``, but as a
|
||||
single dict
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self._result(
|
||||
self._get(self._url("/images/{0}/json", image)), True
|
||||
)
|
||||
|
||||
@utils.minimum_version('1.30')
|
||||
@utils.check_resource('image')
|
||||
def inspect_distribution(self, image, auth_config=None):
|
||||
"""
|
||||
Get image digest and platform information by contacting the registry.
|
||||
|
||||
Args:
|
||||
image (str): The image name to inspect
|
||||
auth_config (dict): Override the credentials that are found in the
|
||||
config for this request. ``auth_config`` should contain the
|
||||
``username`` and ``password`` keys to be valid.
|
||||
|
||||
Returns:
|
||||
(dict): A dict containing distribution data
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
registry, _ = auth.resolve_repository_name(image)
|
||||
|
||||
headers = {}
|
||||
if auth_config is None:
|
||||
header = auth.get_config_header(self, registry)
|
||||
if header:
|
||||
headers['X-Registry-Auth'] = header
|
||||
else:
|
||||
log.debug('Sending supplied auth config')
|
||||
headers['X-Registry-Auth'] = auth.encode_header(auth_config)
|
||||
|
||||
url = self._url("/distribution/{0}/json", image)
|
||||
|
||||
return self._result(
|
||||
self._get(url, headers=headers), True
|
||||
)
|
||||
|
||||
def load_image(self, data, quiet=None):
|
||||
"""
|
||||
Load an image that was previously saved using
|
||||
:py:meth:`~docker.api.image.ImageApiMixin.get_image` (or ``docker
|
||||
save``). Similar to ``docker load``.
|
||||
|
||||
Args:
|
||||
data (binary): Image data to be loaded.
|
||||
quiet (boolean): Suppress progress details in response.
|
||||
|
||||
Returns:
|
||||
(generator): Progress output as JSON objects. Only available for
|
||||
API version >= 1.23
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
params = {}
|
||||
|
||||
if quiet is not None:
|
||||
if utils.version_lt(self._version, '1.23'):
|
||||
raise errors.InvalidVersion(
|
||||
'quiet is not supported in API version < 1.23'
|
||||
)
|
||||
params['quiet'] = quiet
|
||||
|
||||
res = self._post(
|
||||
self._url("/images/load"), data=data, params=params, stream=True
|
||||
)
|
||||
if utils.version_gte(self._version, '1.23'):
|
||||
return self._stream_helper(res, decode=True)
|
||||
|
||||
self._raise_for_status(res)
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
def prune_images(self, filters=None):
|
||||
"""
|
||||
Delete unused images
|
||||
|
||||
Args:
|
||||
filters (dict): Filters to process on the prune list.
|
||||
Available filters:
|
||||
- dangling (bool): When set to true (or 1), prune only
|
||||
unused and untagged images.
|
||||
|
||||
Returns:
|
||||
(dict): A dict containing a list of deleted image IDs and
|
||||
the amount of disk space reclaimed in bytes.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
url = self._url("/images/prune")
|
||||
params = {}
|
||||
if filters is not None:
|
||||
params['filters'] = utils.convert_filters(filters)
|
||||
return self._result(self._post(url, params=params), True)
|
||||
|
||||
def pull(self, repository, tag=None, stream=False, auth_config=None,
|
||||
decode=False, platform=None, all_tags=False):
|
||||
"""
|
||||
Pulls an image. Similar to the ``docker pull`` command.
|
||||
|
||||
Args:
|
||||
repository (str): The repository to pull
|
||||
tag (str): The tag to pull. If ``tag`` is ``None`` or empty, it
|
||||
is set to ``latest``.
|
||||
stream (bool): Stream the output as a generator. Make sure to
|
||||
consume the generator, otherwise pull might get cancelled.
|
||||
auth_config (dict): Override the credentials that are found in the
|
||||
config for this request. ``auth_config`` should contain the
|
||||
``username`` and ``password`` keys to be valid.
|
||||
decode (bool): Decode the JSON data from the server into dicts.
|
||||
Only applies with ``stream=True``
|
||||
platform (str): Platform in the format ``os[/arch[/variant]]``
|
||||
all_tags (bool): Pull all image tags, the ``tag`` parameter is
|
||||
ignored.
|
||||
|
||||
Returns:
|
||||
(generator or str): The output
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> resp = client.api.pull('busybox', stream=True, decode=True)
|
||||
... for line in resp:
|
||||
... print(json.dumps(line, indent=4))
|
||||
{
|
||||
"status": "Pulling image (latest) from busybox",
|
||||
"progressDetail": {},
|
||||
"id": "e72ac664f4f0"
|
||||
}
|
||||
{
|
||||
"status": "Pulling image (latest) from busybox, endpoint: ...",
|
||||
"progressDetail": {},
|
||||
"id": "e72ac664f4f0"
|
||||
}
|
||||
|
||||
"""
|
||||
repository, image_tag = utils.parse_repository_tag(repository)
|
||||
tag = tag or image_tag or 'latest'
|
||||
|
||||
if all_tags:
|
||||
tag = None
|
||||
|
||||
registry, repo_name = auth.resolve_repository_name(repository)
|
||||
|
||||
params = {
|
||||
'tag': tag,
|
||||
'fromImage': repository
|
||||
}
|
||||
headers = {}
|
||||
|
||||
if auth_config is None:
|
||||
header = auth.get_config_header(self, registry)
|
||||
if header:
|
||||
headers['X-Registry-Auth'] = header
|
||||
else:
|
||||
log.debug('Sending supplied auth config')
|
||||
headers['X-Registry-Auth'] = auth.encode_header(auth_config)
|
||||
|
||||
if platform is not None:
|
||||
if utils.version_lt(self._version, '1.32'):
|
||||
raise errors.InvalidVersion(
|
||||
'platform was only introduced in API version 1.32'
|
||||
)
|
||||
params['platform'] = platform
|
||||
|
||||
response = self._post(
|
||||
self._url('/images/create'), params=params, headers=headers,
|
||||
stream=stream, timeout=None
|
||||
)
|
||||
|
||||
self._raise_for_status(response)
|
||||
|
||||
if stream:
|
||||
return self._stream_helper(response, decode=decode)
|
||||
|
||||
return self._result(response)
|
||||
|
||||
def push(self, repository, tag=None, stream=False, auth_config=None,
|
||||
decode=False):
|
||||
"""
|
||||
Push an image or a repository to the registry. Similar to the ``docker
|
||||
push`` command.
|
||||
|
||||
Args:
|
||||
repository (str): The repository to push to
|
||||
tag (str): An optional tag to push
|
||||
stream (bool): Stream the output as a blocking generator
|
||||
auth_config (dict): Override the credentials that are found in the
|
||||
config for this request. ``auth_config`` should contain the
|
||||
``username`` and ``password`` keys to be valid.
|
||||
decode (bool): Decode the JSON data from the server into dicts.
|
||||
Only applies with ``stream=True``
|
||||
|
||||
Returns:
|
||||
(generator or str): The output from the server.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
>>> resp = client.api.push(
|
||||
... 'yourname/app',
|
||||
... stream=True,
|
||||
... decode=True,
|
||||
... )
|
||||
... for line in resp:
|
||||
... print(line)
|
||||
{'status': 'Pushing repository yourname/app (1 tags)'}
|
||||
{'status': 'Pushing','progressDetail': {}, 'id': '511136ea3c5a'}
|
||||
{'status': 'Image already pushed, skipping', 'progressDetail':{},
|
||||
'id': '511136ea3c5a'}
|
||||
...
|
||||
|
||||
"""
|
||||
if not tag:
|
||||
repository, tag = utils.parse_repository_tag(repository)
|
||||
registry, repo_name = auth.resolve_repository_name(repository)
|
||||
u = self._url("/images/{0}/push", repository)
|
||||
params = {
|
||||
'tag': tag
|
||||
}
|
||||
headers = {}
|
||||
|
||||
if auth_config is None:
|
||||
header = auth.get_config_header(self, registry)
|
||||
if header:
|
||||
headers['X-Registry-Auth'] = header
|
||||
else:
|
||||
log.debug('Sending supplied auth config')
|
||||
headers['X-Registry-Auth'] = auth.encode_header(auth_config)
|
||||
|
||||
response = self._post_json(
|
||||
u, None, headers=headers, stream=stream, params=params
|
||||
)
|
||||
|
||||
self._raise_for_status(response)
|
||||
|
||||
if stream:
|
||||
return self._stream_helper(response, decode=decode)
|
||||
|
||||
return self._result(response)
|
||||
|
||||
@utils.check_resource('image')
|
||||
def remove_image(self, image, force=False, noprune=False):
|
||||
"""
|
||||
Remove an image. Similar to the ``docker rmi`` command.
|
||||
|
||||
Args:
|
||||
image (str): The image to remove
|
||||
force (bool): Force removal of the image
|
||||
noprune (bool): Do not delete untagged parents
|
||||
"""
|
||||
params = {'force': force, 'noprune': noprune}
|
||||
res = self._delete(self._url("/images/{0}", image), params=params)
|
||||
return self._result(res, True)
|
||||
|
||||
def search(self, term, limit=None):
|
||||
"""
|
||||
Search for images on Docker Hub. Similar to the ``docker search``
|
||||
command.
|
||||
|
||||
Args:
|
||||
term (str): A term to search for.
|
||||
limit (int): The maximum number of results to return.
|
||||
|
||||
Returns:
|
||||
(list of dicts): The response of the search.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
params = {'term': term}
|
||||
if limit is not None:
|
||||
params['limit'] = limit
|
||||
|
||||
return self._result(
|
||||
self._get(self._url("/images/search"), params=params),
|
||||
True
|
||||
)
|
||||
|
||||
@utils.check_resource('image')
|
||||
def tag(self, image, repository, tag=None, force=False):
|
||||
"""
|
||||
Tag an image into a repository. Similar to the ``docker tag`` command.
|
||||
|
||||
Args:
|
||||
image (str): The image to tag
|
||||
repository (str): The repository to set for the tag
|
||||
tag (str): The tag name
|
||||
force (bool): Force
|
||||
|
||||
Returns:
|
||||
(bool): ``True`` if successful
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> client.api.tag('ubuntu', 'localhost:5000/ubuntu', 'latest',
|
||||
force=True)
|
||||
"""
|
||||
params = {
|
||||
'tag': tag,
|
||||
'repo': repository,
|
||||
'force': 1 if force else 0
|
||||
}
|
||||
url = self._url("/images/{0}/tag", image)
|
||||
res = self._post(url, params=params)
|
||||
self._raise_for_status(res)
|
||||
return res.status_code == 201
|
||||
|
||||
|
||||
def is_file(src):
|
||||
try:
|
||||
return (
|
||||
isinstance(src, str) and
|
||||
os.path.isfile(src)
|
||||
)
|
||||
except TypeError: # a data string will make isfile() raise a TypeError
|
||||
return False
|
||||
|
||||
|
||||
def _import_image_params(repo, tag, image=None, src=None,
|
||||
changes=None):
|
||||
params = {
|
||||
'repo': repo,
|
||||
'tag': tag,
|
||||
}
|
||||
if image:
|
||||
params['fromImage'] = image
|
||||
elif src and not is_file(src):
|
||||
params['fromSrc'] = src
|
||||
else:
|
||||
params['fromSrc'] = '-'
|
||||
|
||||
if changes:
|
||||
params['changes'] = changes
|
||||
|
||||
return params
|
||||
@@ -0,0 +1,277 @@
|
||||
from .. import utils
|
||||
from ..errors import InvalidVersion
|
||||
from ..utils import check_resource, minimum_version, version_lt
|
||||
|
||||
|
||||
class NetworkApiMixin:
|
||||
def networks(self, names=None, ids=None, filters=None):
|
||||
"""
|
||||
List networks. Similar to the ``docker network ls`` command.
|
||||
|
||||
Args:
|
||||
names (:py:class:`list`): List of names to filter by
|
||||
ids (:py:class:`list`): List of ids to filter by
|
||||
filters (dict): Filters to be processed on the network list.
|
||||
Available filters:
|
||||
- ``driver=[<driver-name>]`` Matches a network's driver.
|
||||
- ``label=[<key>]``, ``label=[<key>=<value>]`` or a list of
|
||||
such.
|
||||
- ``type=["custom"|"builtin"]`` Filters networks by type.
|
||||
|
||||
Returns:
|
||||
(dict): List of network objects.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
|
||||
if filters is None:
|
||||
filters = {}
|
||||
if names:
|
||||
filters['name'] = names
|
||||
if ids:
|
||||
filters['id'] = ids
|
||||
params = {'filters': utils.convert_filters(filters)}
|
||||
url = self._url("/networks")
|
||||
res = self._get(url, params=params)
|
||||
return self._result(res, json=True)
|
||||
|
||||
def create_network(self, name, driver=None, options=None, ipam=None,
|
||||
check_duplicate=None, internal=False, labels=None,
|
||||
enable_ipv6=False, attachable=None, scope=None,
|
||||
ingress=None):
|
||||
"""
|
||||
Create a network. Similar to the ``docker network create``.
|
||||
|
||||
Args:
|
||||
name (str): Name of the network
|
||||
driver (str): Name of the driver used to create the network
|
||||
options (dict): Driver options as a key-value dictionary
|
||||
ipam (IPAMConfig): Optional custom IP scheme for the network.
|
||||
check_duplicate (bool): Request daemon to check for networks with
|
||||
same name. Default: ``None``.
|
||||
internal (bool): Restrict external access to the network. Default
|
||||
``False``.
|
||||
labels (dict): Map of labels to set on the network. Default
|
||||
``None``.
|
||||
enable_ipv6 (bool): Enable IPv6 on the network. Default ``False``.
|
||||
attachable (bool): If enabled, and the network is in the global
|
||||
scope, non-service containers on worker nodes will be able to
|
||||
connect to the network.
|
||||
scope (str): Specify the network's scope (``local``, ``global`` or
|
||||
``swarm``)
|
||||
ingress (bool): If set, create an ingress network which provides
|
||||
the routing-mesh in swarm mode.
|
||||
|
||||
Returns:
|
||||
(dict): The created network reference object
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
A network using the bridge driver:
|
||||
|
||||
>>> client.api.create_network("network1", driver="bridge")
|
||||
|
||||
You can also create more advanced networks with custom IPAM
|
||||
configurations. For example, setting the subnet to
|
||||
``192.168.52.0/24`` and gateway address to ``192.168.52.254``.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
>>> ipam_pool = docker.types.IPAMPool(
|
||||
subnet='192.168.52.0/24',
|
||||
gateway='192.168.52.254'
|
||||
)
|
||||
>>> ipam_config = docker.types.IPAMConfig(
|
||||
pool_configs=[ipam_pool]
|
||||
)
|
||||
>>> client.api.create_network("network1", driver="bridge",
|
||||
ipam=ipam_config)
|
||||
"""
|
||||
if options is not None and not isinstance(options, dict):
|
||||
raise TypeError('options must be a dictionary')
|
||||
|
||||
data = {
|
||||
'Name': name,
|
||||
'Driver': driver,
|
||||
'Options': options,
|
||||
'IPAM': ipam,
|
||||
'CheckDuplicate': check_duplicate,
|
||||
}
|
||||
|
||||
if labels is not None:
|
||||
if version_lt(self._version, '1.23'):
|
||||
raise InvalidVersion(
|
||||
'network labels were introduced in API 1.23'
|
||||
)
|
||||
if not isinstance(labels, dict):
|
||||
raise TypeError('labels must be a dictionary')
|
||||
data["Labels"] = labels
|
||||
|
||||
if enable_ipv6:
|
||||
if version_lt(self._version, '1.23'):
|
||||
raise InvalidVersion(
|
||||
'enable_ipv6 was introduced in API 1.23'
|
||||
)
|
||||
data['EnableIPv6'] = True
|
||||
|
||||
if internal:
|
||||
if version_lt(self._version, '1.22'):
|
||||
raise InvalidVersion('Internal networks are not '
|
||||
'supported in API version < 1.22')
|
||||
data['Internal'] = True
|
||||
|
||||
if attachable is not None:
|
||||
if version_lt(self._version, '1.24'):
|
||||
raise InvalidVersion(
|
||||
'attachable is not supported in API version < 1.24'
|
||||
)
|
||||
data['Attachable'] = attachable
|
||||
|
||||
if ingress is not None:
|
||||
if version_lt(self._version, '1.29'):
|
||||
raise InvalidVersion(
|
||||
'ingress is not supported in API version < 1.29'
|
||||
)
|
||||
|
||||
data['Ingress'] = ingress
|
||||
|
||||
if scope is not None:
|
||||
if version_lt(self._version, '1.30'):
|
||||
raise InvalidVersion(
|
||||
'scope is not supported in API version < 1.30'
|
||||
)
|
||||
data['Scope'] = scope
|
||||
|
||||
url = self._url("/networks/create")
|
||||
res = self._post_json(url, data=data)
|
||||
return self._result(res, json=True)
|
||||
|
||||
@minimum_version('1.25')
|
||||
def prune_networks(self, filters=None):
|
||||
"""
|
||||
Delete unused networks
|
||||
|
||||
Args:
|
||||
filters (dict): Filters to process on the prune list.
|
||||
|
||||
Returns:
|
||||
(dict): A dict containing a list of deleted network names and
|
||||
the amount of disk space reclaimed in bytes.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
params = {}
|
||||
if filters:
|
||||
params['filters'] = utils.convert_filters(filters)
|
||||
url = self._url('/networks/prune')
|
||||
return self._result(self._post(url, params=params), True)
|
||||
|
||||
@check_resource('net_id')
|
||||
def remove_network(self, net_id):
|
||||
"""
|
||||
Remove a network. Similar to the ``docker network rm`` command.
|
||||
|
||||
Args:
|
||||
net_id (str): The network's id
|
||||
"""
|
||||
url = self._url("/networks/{0}", net_id)
|
||||
res = self._delete(url)
|
||||
self._raise_for_status(res)
|
||||
|
||||
@check_resource('net_id')
|
||||
def inspect_network(self, net_id, verbose=None, scope=None):
|
||||
"""
|
||||
Get detailed information about a network.
|
||||
|
||||
Args:
|
||||
net_id (str): ID of network
|
||||
verbose (bool): Show the service details across the cluster in
|
||||
swarm mode.
|
||||
scope (str): Filter the network by scope (``swarm``, ``global``
|
||||
or ``local``).
|
||||
"""
|
||||
params = {}
|
||||
if verbose is not None:
|
||||
if version_lt(self._version, '1.28'):
|
||||
raise InvalidVersion('verbose was introduced in API 1.28')
|
||||
params['verbose'] = verbose
|
||||
if scope is not None:
|
||||
if version_lt(self._version, '1.31'):
|
||||
raise InvalidVersion('scope was introduced in API 1.31')
|
||||
params['scope'] = scope
|
||||
|
||||
url = self._url("/networks/{0}", net_id)
|
||||
res = self._get(url, params=params)
|
||||
return self._result(res, json=True)
|
||||
|
||||
@check_resource('container')
|
||||
def connect_container_to_network(self, container, net_id,
|
||||
ipv4_address=None, ipv6_address=None,
|
||||
aliases=None, links=None,
|
||||
link_local_ips=None, driver_opt=None,
|
||||
mac_address=None):
|
||||
"""
|
||||
Connect a container to a network.
|
||||
|
||||
Args:
|
||||
container (str): container-id/name to be connected to the network
|
||||
net_id (str): network id
|
||||
aliases (:py:class:`list`): A list of aliases for this endpoint.
|
||||
Names in that list can be used within the network to reach the
|
||||
container. Defaults to ``None``.
|
||||
links (:py:class:`list`): A list of links for this endpoint.
|
||||
Containers declared in this list will be linked to this
|
||||
container. Defaults to ``None``.
|
||||
ipv4_address (str): The IP address of this container on the
|
||||
network, using the IPv4 protocol. Defaults to ``None``.
|
||||
ipv6_address (str): The IP address of this container on the
|
||||
network, using the IPv6 protocol. Defaults to ``None``.
|
||||
link_local_ips (:py:class:`list`): A list of link-local
|
||||
(IPv4/IPv6) addresses.
|
||||
mac_address (str): The MAC address of this container on the
|
||||
network. Defaults to ``None``.
|
||||
"""
|
||||
data = {
|
||||
"Container": container,
|
||||
"EndpointConfig": self.create_endpoint_config(
|
||||
aliases=aliases, links=links, ipv4_address=ipv4_address,
|
||||
ipv6_address=ipv6_address, link_local_ips=link_local_ips,
|
||||
driver_opt=driver_opt,
|
||||
mac_address=mac_address
|
||||
),
|
||||
}
|
||||
|
||||
url = self._url("/networks/{0}/connect", net_id)
|
||||
res = self._post_json(url, data=data)
|
||||
self._raise_for_status(res)
|
||||
|
||||
@check_resource('container')
|
||||
def disconnect_container_from_network(self, container, net_id,
|
||||
force=False):
|
||||
"""
|
||||
Disconnect a container from a network.
|
||||
|
||||
Args:
|
||||
container (str): container ID or name to be disconnected from the
|
||||
network
|
||||
net_id (str): network ID
|
||||
force (bool): Force the container to disconnect from a network.
|
||||
Default: ``False``
|
||||
"""
|
||||
data = {"Container": container}
|
||||
if force:
|
||||
if version_lt(self._version, '1.22'):
|
||||
raise InvalidVersion(
|
||||
'Forced disconnect was introduced in API 1.22'
|
||||
)
|
||||
data['Force'] = force
|
||||
url = self._url("/networks/{0}/disconnect", net_id)
|
||||
res = self._post_json(url, data=data)
|
||||
self._raise_for_status(res)
|
||||
@@ -0,0 +1,261 @@
|
||||
from .. import auth, utils
|
||||
|
||||
|
||||
class PluginApiMixin:
|
||||
@utils.minimum_version('1.25')
|
||||
@utils.check_resource('name')
|
||||
def configure_plugin(self, name, options):
|
||||
"""
|
||||
Configure a plugin.
|
||||
|
||||
Args:
|
||||
name (string): The name of the plugin. The ``:latest`` tag is
|
||||
optional, and is the default if omitted.
|
||||
options (dict): A key-value mapping of options
|
||||
|
||||
Returns:
|
||||
``True`` if successful
|
||||
"""
|
||||
url = self._url('/plugins/{0}/set', name)
|
||||
data = options
|
||||
if isinstance(data, dict):
|
||||
data = [f'{k}={v}' for k, v in data.items()]
|
||||
res = self._post_json(url, data=data)
|
||||
self._raise_for_status(res)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
def create_plugin(self, name, plugin_data_dir, gzip=False):
|
||||
"""
|
||||
Create a new plugin.
|
||||
|
||||
Args:
|
||||
name (string): The name of the plugin. The ``:latest`` tag is
|
||||
optional, and is the default if omitted.
|
||||
plugin_data_dir (string): Path to the plugin data directory.
|
||||
Plugin data directory must contain the ``config.json``
|
||||
manifest file and the ``rootfs`` directory.
|
||||
gzip (bool): Compress the context using gzip. Default: False
|
||||
|
||||
Returns:
|
||||
``True`` if successful
|
||||
"""
|
||||
url = self._url('/plugins/create')
|
||||
|
||||
with utils.create_archive(
|
||||
root=plugin_data_dir, gzip=gzip,
|
||||
files=set(utils.build.walk(plugin_data_dir, []))
|
||||
) as archv:
|
||||
res = self._post(url, params={'name': name}, data=archv)
|
||||
self._raise_for_status(res)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
def disable_plugin(self, name, force=False):
|
||||
"""
|
||||
Disable an installed plugin.
|
||||
|
||||
Args:
|
||||
name (string): The name of the plugin. The ``:latest`` tag is
|
||||
optional, and is the default if omitted.
|
||||
force (bool): To enable the force query parameter.
|
||||
|
||||
Returns:
|
||||
``True`` if successful
|
||||
"""
|
||||
url = self._url('/plugins/{0}/disable', name)
|
||||
res = self._post(url, params={'force': force})
|
||||
self._raise_for_status(res)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
def enable_plugin(self, name, timeout=0):
|
||||
"""
|
||||
Enable an installed plugin.
|
||||
|
||||
Args:
|
||||
name (string): The name of the plugin. The ``:latest`` tag is
|
||||
optional, and is the default if omitted.
|
||||
timeout (int): Operation timeout (in seconds). Default: 0
|
||||
|
||||
Returns:
|
||||
``True`` if successful
|
||||
"""
|
||||
url = self._url('/plugins/{0}/enable', name)
|
||||
params = {'timeout': timeout}
|
||||
res = self._post(url, params=params)
|
||||
self._raise_for_status(res)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
def inspect_plugin(self, name):
|
||||
"""
|
||||
Retrieve plugin metadata.
|
||||
|
||||
Args:
|
||||
name (string): The name of the plugin. The ``:latest`` tag is
|
||||
optional, and is the default if omitted.
|
||||
|
||||
Returns:
|
||||
A dict containing plugin info
|
||||
"""
|
||||
url = self._url('/plugins/{0}/json', name)
|
||||
return self._result(self._get(url), True)
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
def pull_plugin(self, remote, privileges, name=None):
|
||||
"""
|
||||
Pull and install a plugin. After the plugin is installed, it can be
|
||||
enabled using :py:meth:`~enable_plugin`.
|
||||
|
||||
Args:
|
||||
remote (string): Remote reference for the plugin to install.
|
||||
The ``:latest`` tag is optional, and is the default if
|
||||
omitted.
|
||||
privileges (:py:class:`list`): A list of privileges the user
|
||||
consents to grant to the plugin. Can be retrieved using
|
||||
:py:meth:`~plugin_privileges`.
|
||||
name (string): Local name for the pulled plugin. The
|
||||
``:latest`` tag is optional, and is the default if omitted.
|
||||
|
||||
Returns:
|
||||
An iterable object streaming the decoded API logs
|
||||
"""
|
||||
url = self._url('/plugins/pull')
|
||||
params = {
|
||||
'remote': remote,
|
||||
}
|
||||
if name:
|
||||
params['name'] = name
|
||||
|
||||
headers = {}
|
||||
registry, repo_name = auth.resolve_repository_name(remote)
|
||||
header = auth.get_config_header(self, registry)
|
||||
if header:
|
||||
headers['X-Registry-Auth'] = header
|
||||
response = self._post_json(
|
||||
url, params=params, headers=headers, data=privileges,
|
||||
stream=True
|
||||
)
|
||||
self._raise_for_status(response)
|
||||
return self._stream_helper(response, decode=True)
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
def plugins(self):
|
||||
"""
|
||||
Retrieve a list of installed plugins.
|
||||
|
||||
Returns:
|
||||
A list of dicts, one per plugin
|
||||
"""
|
||||
url = self._url('/plugins')
|
||||
return self._result(self._get(url), True)
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
def plugin_privileges(self, name):
|
||||
"""
|
||||
Retrieve list of privileges to be granted to a plugin.
|
||||
|
||||
Args:
|
||||
name (string): Name of the remote plugin to examine. The
|
||||
``:latest`` tag is optional, and is the default if omitted.
|
||||
|
||||
Returns:
|
||||
A list of dictionaries representing the plugin's
|
||||
permissions
|
||||
|
||||
"""
|
||||
params = {
|
||||
'remote': name,
|
||||
}
|
||||
|
||||
headers = {}
|
||||
registry, repo_name = auth.resolve_repository_name(name)
|
||||
header = auth.get_config_header(self, registry)
|
||||
if header:
|
||||
headers['X-Registry-Auth'] = header
|
||||
|
||||
url = self._url('/plugins/privileges')
|
||||
return self._result(
|
||||
self._get(url, params=params, headers=headers), True
|
||||
)
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
@utils.check_resource('name')
|
||||
def push_plugin(self, name):
|
||||
"""
|
||||
Push a plugin to the registry.
|
||||
|
||||
Args:
|
||||
name (string): Name of the plugin to upload. The ``:latest``
|
||||
tag is optional, and is the default if omitted.
|
||||
|
||||
Returns:
|
||||
``True`` if successful
|
||||
"""
|
||||
url = self._url('/plugins/{0}/pull', name)
|
||||
|
||||
headers = {}
|
||||
registry, repo_name = auth.resolve_repository_name(name)
|
||||
header = auth.get_config_header(self, registry)
|
||||
if header:
|
||||
headers['X-Registry-Auth'] = header
|
||||
res = self._post(url, headers=headers)
|
||||
self._raise_for_status(res)
|
||||
return self._stream_helper(res, decode=True)
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
@utils.check_resource('name')
|
||||
def remove_plugin(self, name, force=False):
|
||||
"""
|
||||
Remove an installed plugin.
|
||||
|
||||
Args:
|
||||
name (string): Name of the plugin to remove. The ``:latest``
|
||||
tag is optional, and is the default if omitted.
|
||||
force (bool): Disable the plugin before removing. This may
|
||||
result in issues if the plugin is in use by a container.
|
||||
|
||||
Returns:
|
||||
``True`` if successful
|
||||
"""
|
||||
url = self._url('/plugins/{0}', name)
|
||||
res = self._delete(url, params={'force': force})
|
||||
self._raise_for_status(res)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.26')
|
||||
@utils.check_resource('name')
|
||||
def upgrade_plugin(self, name, remote, privileges):
|
||||
"""
|
||||
Upgrade an installed plugin.
|
||||
|
||||
Args:
|
||||
name (string): Name of the plugin to upgrade. The ``:latest``
|
||||
tag is optional and is the default if omitted.
|
||||
remote (string): Remote reference to upgrade to. The
|
||||
``:latest`` tag is optional and is the default if omitted.
|
||||
privileges (:py:class:`list`): A list of privileges the user
|
||||
consents to grant to the plugin. Can be retrieved using
|
||||
:py:meth:`~plugin_privileges`.
|
||||
|
||||
Returns:
|
||||
An iterable object streaming the decoded API logs
|
||||
"""
|
||||
|
||||
url = self._url('/plugins/{0}/upgrade', name)
|
||||
params = {
|
||||
'remote': remote,
|
||||
}
|
||||
|
||||
headers = {}
|
||||
registry, repo_name = auth.resolve_repository_name(remote)
|
||||
header = auth.get_config_header(self, registry)
|
||||
if header:
|
||||
headers['X-Registry-Auth'] = header
|
||||
response = self._post_json(
|
||||
url, params=params, headers=headers, data=privileges,
|
||||
stream=True
|
||||
)
|
||||
self._raise_for_status(response)
|
||||
return self._stream_helper(response, decode=True)
|
||||
@@ -0,0 +1,98 @@
|
||||
import base64
|
||||
|
||||
from .. import errors, utils
|
||||
|
||||
|
||||
class SecretApiMixin:
|
||||
@utils.minimum_version('1.25')
|
||||
def create_secret(self, name, data, labels=None, driver=None):
|
||||
"""
|
||||
Create a secret
|
||||
|
||||
Args:
|
||||
name (string): Name of the secret
|
||||
data (bytes): Secret data to be stored
|
||||
labels (dict): A mapping of labels to assign to the secret
|
||||
driver (DriverConfig): A custom driver configuration. If
|
||||
unspecified, the default ``internal`` driver will be used
|
||||
|
||||
Returns (dict): ID of the newly created secret
|
||||
"""
|
||||
if not isinstance(data, bytes):
|
||||
data = data.encode('utf-8')
|
||||
|
||||
data = base64.b64encode(data)
|
||||
data = data.decode('ascii')
|
||||
body = {
|
||||
'Data': data,
|
||||
'Name': name,
|
||||
'Labels': labels
|
||||
}
|
||||
|
||||
if driver is not None:
|
||||
if utils.version_lt(self._version, '1.31'):
|
||||
raise errors.InvalidVersion(
|
||||
'Secret driver is only available for API version > 1.31'
|
||||
)
|
||||
|
||||
body['Driver'] = driver
|
||||
|
||||
url = self._url('/secrets/create')
|
||||
return self._result(
|
||||
self._post_json(url, data=body), True
|
||||
)
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
@utils.check_resource('id')
|
||||
def inspect_secret(self, id):
|
||||
"""
|
||||
Retrieve secret metadata
|
||||
|
||||
Args:
|
||||
id (string): Full ID of the secret to inspect
|
||||
|
||||
Returns (dict): A dictionary of metadata
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.NotFound`
|
||||
if no secret with that ID exists
|
||||
"""
|
||||
url = self._url('/secrets/{0}', id)
|
||||
return self._result(self._get(url), True)
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
@utils.check_resource('id')
|
||||
def remove_secret(self, id):
|
||||
"""
|
||||
Remove a secret
|
||||
|
||||
Args:
|
||||
id (string): Full ID of the secret to remove
|
||||
|
||||
Returns (boolean): True if successful
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.NotFound`
|
||||
if no secret with that ID exists
|
||||
"""
|
||||
url = self._url('/secrets/{0}', id)
|
||||
res = self._delete(url)
|
||||
self._raise_for_status(res)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
def secrets(self, filters=None):
|
||||
"""
|
||||
List secrets
|
||||
|
||||
Args:
|
||||
filters (dict): A map of filters to process on the secrets
|
||||
list. Available filters: ``names``
|
||||
|
||||
Returns (list): A list of secrets
|
||||
"""
|
||||
url = self._url('/secrets')
|
||||
params = {}
|
||||
if filters:
|
||||
params['filters'] = utils.convert_filters(filters)
|
||||
return self._result(self._get(url, params=params), True)
|
||||
@@ -0,0 +1,486 @@
|
||||
from .. import auth, errors, utils
|
||||
from ..types import ServiceMode
|
||||
|
||||
|
||||
def _check_api_features(version, task_template, update_config, endpoint_spec,
|
||||
rollback_config):
|
||||
|
||||
def raise_version_error(param, min_version):
|
||||
raise errors.InvalidVersion(
|
||||
f'{param} is not supported in API version < {min_version}'
|
||||
)
|
||||
|
||||
if update_config is not None:
|
||||
if utils.version_lt(version, '1.25'):
|
||||
if 'MaxFailureRatio' in update_config:
|
||||
raise_version_error('UpdateConfig.max_failure_ratio', '1.25')
|
||||
if 'Monitor' in update_config:
|
||||
raise_version_error('UpdateConfig.monitor', '1.25')
|
||||
|
||||
if utils.version_lt(version, '1.28'):
|
||||
if update_config.get('FailureAction') == 'rollback':
|
||||
raise_version_error(
|
||||
'UpdateConfig.failure_action rollback', '1.28'
|
||||
)
|
||||
|
||||
if utils.version_lt(version, '1.29'):
|
||||
if 'Order' in update_config:
|
||||
raise_version_error('UpdateConfig.order', '1.29')
|
||||
|
||||
if rollback_config is not None:
|
||||
if utils.version_lt(version, '1.28'):
|
||||
raise_version_error('rollback_config', '1.28')
|
||||
|
||||
if utils.version_lt(version, '1.29'):
|
||||
if 'Order' in update_config:
|
||||
raise_version_error('RollbackConfig.order', '1.29')
|
||||
|
||||
if endpoint_spec is not None:
|
||||
if utils.version_lt(version, '1.32') and 'Ports' in endpoint_spec:
|
||||
if any(p.get('PublishMode') for p in endpoint_spec['Ports']):
|
||||
raise_version_error('EndpointSpec.Ports[].mode', '1.32')
|
||||
|
||||
if task_template is not None:
|
||||
if 'ForceUpdate' in task_template and utils.version_lt(
|
||||
version, '1.25'):
|
||||
raise_version_error('force_update', '1.25')
|
||||
|
||||
if task_template.get('Placement'):
|
||||
if utils.version_lt(version, '1.30'):
|
||||
if task_template['Placement'].get('Platforms'):
|
||||
raise_version_error('Placement.platforms', '1.30')
|
||||
if utils.version_lt(version, '1.27'):
|
||||
if task_template['Placement'].get('Preferences'):
|
||||
raise_version_error('Placement.preferences', '1.27')
|
||||
|
||||
if task_template.get('ContainerSpec'):
|
||||
container_spec = task_template.get('ContainerSpec')
|
||||
|
||||
if utils.version_lt(version, '1.25'):
|
||||
if container_spec.get('TTY'):
|
||||
raise_version_error('ContainerSpec.tty', '1.25')
|
||||
if container_spec.get('Hostname') is not None:
|
||||
raise_version_error('ContainerSpec.hostname', '1.25')
|
||||
if container_spec.get('Hosts') is not None:
|
||||
raise_version_error('ContainerSpec.hosts', '1.25')
|
||||
if container_spec.get('Groups') is not None:
|
||||
raise_version_error('ContainerSpec.groups', '1.25')
|
||||
if container_spec.get('DNSConfig') is not None:
|
||||
raise_version_error('ContainerSpec.dns_config', '1.25')
|
||||
if container_spec.get('Healthcheck') is not None:
|
||||
raise_version_error('ContainerSpec.healthcheck', '1.25')
|
||||
|
||||
if utils.version_lt(version, '1.28'):
|
||||
if container_spec.get('ReadOnly') is not None:
|
||||
raise_version_error('ContainerSpec.dns_config', '1.28')
|
||||
if container_spec.get('StopSignal') is not None:
|
||||
raise_version_error('ContainerSpec.stop_signal', '1.28')
|
||||
|
||||
if utils.version_lt(version, '1.30'):
|
||||
if container_spec.get('Configs') is not None:
|
||||
raise_version_error('ContainerSpec.configs', '1.30')
|
||||
if container_spec.get('Privileges') is not None:
|
||||
raise_version_error('ContainerSpec.privileges', '1.30')
|
||||
|
||||
if utils.version_lt(version, '1.35'):
|
||||
if container_spec.get('Isolation') is not None:
|
||||
raise_version_error('ContainerSpec.isolation', '1.35')
|
||||
|
||||
if utils.version_lt(version, '1.38'):
|
||||
if container_spec.get('Init') is not None:
|
||||
raise_version_error('ContainerSpec.init', '1.38')
|
||||
|
||||
if task_template.get('Resources'):
|
||||
if utils.version_lt(version, '1.32'):
|
||||
if task_template['Resources'].get('GenericResources'):
|
||||
raise_version_error('Resources.generic_resources', '1.32')
|
||||
|
||||
|
||||
def _merge_task_template(current, override):
|
||||
merged = current.copy()
|
||||
if override is not None:
|
||||
for ts_key, ts_value in override.items():
|
||||
if ts_key == 'ContainerSpec':
|
||||
if 'ContainerSpec' not in merged:
|
||||
merged['ContainerSpec'] = {}
|
||||
for cs_key, cs_value in override['ContainerSpec'].items():
|
||||
if cs_value is not None:
|
||||
merged['ContainerSpec'][cs_key] = cs_value
|
||||
elif ts_value is not None:
|
||||
merged[ts_key] = ts_value
|
||||
return merged
|
||||
|
||||
|
||||
class ServiceApiMixin:
|
||||
@utils.minimum_version('1.24')
|
||||
def create_service(
|
||||
self, task_template, name=None, labels=None, mode=None,
|
||||
update_config=None, networks=None, endpoint_config=None,
|
||||
endpoint_spec=None, rollback_config=None
|
||||
):
|
||||
"""
|
||||
Create a service.
|
||||
|
||||
Args:
|
||||
task_template (TaskTemplate): Specification of the task to start as
|
||||
part of the new service.
|
||||
name (string): User-defined name for the service. Optional.
|
||||
labels (dict): A map of labels to associate with the service.
|
||||
Optional.
|
||||
mode (ServiceMode): Scheduling mode for the service (replicated
|
||||
or global). Defaults to replicated.
|
||||
update_config (UpdateConfig): Specification for the update strategy
|
||||
of the service. Default: ``None``
|
||||
rollback_config (RollbackConfig): Specification for the rollback
|
||||
strategy of the service. Default: ``None``
|
||||
networks (:py:class:`list`): List of network names or IDs or
|
||||
:py:class:`~docker.types.NetworkAttachmentConfig` to attach the
|
||||
service to. Default: ``None``.
|
||||
endpoint_spec (EndpointSpec): Properties that can be configured to
|
||||
access and load balance a service. Default: ``None``.
|
||||
|
||||
Returns:
|
||||
A dictionary containing an ``ID`` key for the newly created
|
||||
service.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
|
||||
_check_api_features(
|
||||
self._version, task_template, update_config, endpoint_spec,
|
||||
rollback_config
|
||||
)
|
||||
|
||||
url = self._url('/services/create')
|
||||
headers = {}
|
||||
image = task_template.get('ContainerSpec', {}).get('Image', None)
|
||||
if image is None:
|
||||
raise errors.DockerException(
|
||||
'Missing mandatory Image key in ContainerSpec'
|
||||
)
|
||||
if mode and not isinstance(mode, dict):
|
||||
mode = ServiceMode(mode)
|
||||
|
||||
registry, repo_name = auth.resolve_repository_name(image)
|
||||
auth_header = auth.get_config_header(self, registry)
|
||||
if auth_header:
|
||||
headers['X-Registry-Auth'] = auth_header
|
||||
if utils.version_lt(self._version, '1.25'):
|
||||
networks = networks or task_template.pop('Networks', None)
|
||||
data = {
|
||||
'Name': name,
|
||||
'Labels': labels,
|
||||
'TaskTemplate': task_template,
|
||||
'Mode': mode,
|
||||
'Networks': utils.convert_service_networks(networks),
|
||||
'EndpointSpec': endpoint_spec
|
||||
}
|
||||
|
||||
if update_config is not None:
|
||||
data['UpdateConfig'] = update_config
|
||||
|
||||
if rollback_config is not None:
|
||||
data['RollbackConfig'] = rollback_config
|
||||
|
||||
return self._result(
|
||||
self._post_json(url, data=data, headers=headers), True
|
||||
)
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
@utils.check_resource('service')
|
||||
def inspect_service(self, service, insert_defaults=None):
|
||||
"""
|
||||
Return information about a service.
|
||||
|
||||
Args:
|
||||
service (str): Service name or ID.
|
||||
insert_defaults (boolean): If true, default values will be merged
|
||||
into the service inspect output.
|
||||
|
||||
Returns:
|
||||
(dict): A dictionary of the server-side representation of the
|
||||
service, including all relevant properties.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
url = self._url('/services/{0}', service)
|
||||
params = {}
|
||||
if insert_defaults is not None:
|
||||
if utils.version_lt(self._version, '1.29'):
|
||||
raise errors.InvalidVersion(
|
||||
'insert_defaults is not supported in API version < 1.29'
|
||||
)
|
||||
params['insertDefaults'] = insert_defaults
|
||||
|
||||
return self._result(self._get(url, params=params), True)
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
@utils.check_resource('task')
|
||||
def inspect_task(self, task):
|
||||
"""
|
||||
Retrieve information about a task.
|
||||
|
||||
Args:
|
||||
task (str): Task ID
|
||||
|
||||
Returns:
|
||||
(dict): Information about the task.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
url = self._url('/tasks/{0}', task)
|
||||
return self._result(self._get(url), True)
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
@utils.check_resource('service')
|
||||
def remove_service(self, service):
|
||||
"""
|
||||
Stop and remove a service.
|
||||
|
||||
Args:
|
||||
service (str): Service name or ID
|
||||
|
||||
Returns:
|
||||
``True`` if successful.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
|
||||
url = self._url('/services/{0}', service)
|
||||
resp = self._delete(url)
|
||||
self._raise_for_status(resp)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
def services(self, filters=None, status=None):
|
||||
"""
|
||||
List services.
|
||||
|
||||
Args:
|
||||
filters (dict): Filters to process on the nodes list. Valid
|
||||
filters: ``id``, ``name`` , ``label`` and ``mode``.
|
||||
Default: ``None``.
|
||||
status (bool): Include the service task count of running and
|
||||
desired tasks. Default: ``None``.
|
||||
|
||||
Returns:
|
||||
A list of dictionaries containing data about each service.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
params = {
|
||||
'filters': utils.convert_filters(filters) if filters else None
|
||||
}
|
||||
if status is not None:
|
||||
if utils.version_lt(self._version, '1.41'):
|
||||
raise errors.InvalidVersion(
|
||||
'status is not supported in API version < 1.41'
|
||||
)
|
||||
params['status'] = status
|
||||
url = self._url('/services')
|
||||
return self._result(self._get(url, params=params), True)
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
@utils.check_resource('service')
|
||||
def service_logs(self, service, details=False, follow=False, stdout=False,
|
||||
stderr=False, since=0, timestamps=False, tail='all',
|
||||
is_tty=None):
|
||||
"""
|
||||
Get log stream for a service.
|
||||
Note: This endpoint works only for services with the ``json-file``
|
||||
or ``journald`` logging drivers.
|
||||
|
||||
Args:
|
||||
service (str): ID or name of the service
|
||||
details (bool): Show extra details provided to logs.
|
||||
Default: ``False``
|
||||
follow (bool): Keep connection open to read logs as they are
|
||||
sent by the Engine. Default: ``False``
|
||||
stdout (bool): Return logs from ``stdout``. Default: ``False``
|
||||
stderr (bool): Return logs from ``stderr``. Default: ``False``
|
||||
since (int): UNIX timestamp for the logs staring point.
|
||||
Default: 0
|
||||
timestamps (bool): Add timestamps to every log line.
|
||||
tail (string or int): Number of log lines to be returned,
|
||||
counting from the current end of the logs. Specify an
|
||||
integer or ``'all'`` to output all log lines.
|
||||
Default: ``all``
|
||||
is_tty (bool): Whether the service's :py:class:`ContainerSpec`
|
||||
enables the TTY option. If omitted, the method will query
|
||||
the Engine for the information, causing an additional
|
||||
roundtrip.
|
||||
|
||||
Returns (generator): Logs for the service.
|
||||
"""
|
||||
params = {
|
||||
'details': details,
|
||||
'follow': follow,
|
||||
'stdout': stdout,
|
||||
'stderr': stderr,
|
||||
'since': since,
|
||||
'timestamps': timestamps,
|
||||
'tail': tail
|
||||
}
|
||||
|
||||
url = self._url('/services/{0}/logs', service)
|
||||
res = self._get(url, params=params, stream=True)
|
||||
if is_tty is None:
|
||||
is_tty = self.inspect_service(
|
||||
service
|
||||
)['Spec']['TaskTemplate']['ContainerSpec'].get('TTY', False)
|
||||
return self._get_result_tty(True, res, is_tty)
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
def tasks(self, filters=None):
|
||||
"""
|
||||
Retrieve a list of tasks.
|
||||
|
||||
Args:
|
||||
filters (dict): A map of filters to process on the tasks list.
|
||||
Valid filters: ``id``, ``name``, ``service``, ``node``,
|
||||
``label`` and ``desired-state``.
|
||||
|
||||
Returns:
|
||||
(:py:class:`list`): List of task dictionaries.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
|
||||
params = {
|
||||
'filters': utils.convert_filters(filters) if filters else None
|
||||
}
|
||||
url = self._url('/tasks')
|
||||
return self._result(self._get(url, params=params), True)
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
@utils.check_resource('service')
|
||||
def update_service(self, service, version, task_template=None, name=None,
|
||||
labels=None, mode=None, update_config=None,
|
||||
networks=None, endpoint_config=None,
|
||||
endpoint_spec=None, fetch_current_spec=False,
|
||||
rollback_config=None):
|
||||
"""
|
||||
Update a service.
|
||||
|
||||
Args:
|
||||
service (string): A service identifier (either its name or service
|
||||
ID).
|
||||
version (int): The version number of the service object being
|
||||
updated. This is required to avoid conflicting writes.
|
||||
task_template (TaskTemplate): Specification of the updated task to
|
||||
start as part of the service.
|
||||
name (string): New name for the service. Optional.
|
||||
labels (dict): A map of labels to associate with the service.
|
||||
Optional.
|
||||
mode (ServiceMode): Scheduling mode for the service (replicated
|
||||
or global). Defaults to replicated.
|
||||
update_config (UpdateConfig): Specification for the update strategy
|
||||
of the service. Default: ``None``.
|
||||
rollback_config (RollbackConfig): Specification for the rollback
|
||||
strategy of the service. Default: ``None``
|
||||
networks (:py:class:`list`): List of network names or IDs or
|
||||
:py:class:`~docker.types.NetworkAttachmentConfig` to attach the
|
||||
service to. Default: ``None``.
|
||||
endpoint_spec (EndpointSpec): Properties that can be configured to
|
||||
access and load balance a service. Default: ``None``.
|
||||
fetch_current_spec (boolean): Use the undefined settings from the
|
||||
current specification of the service. Default: ``False``
|
||||
|
||||
Returns:
|
||||
A dictionary containing a ``Warnings`` key.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
|
||||
_check_api_features(
|
||||
self._version, task_template, update_config, endpoint_spec,
|
||||
rollback_config
|
||||
)
|
||||
|
||||
if fetch_current_spec:
|
||||
inspect_defaults = True
|
||||
if utils.version_lt(self._version, '1.29'):
|
||||
inspect_defaults = None
|
||||
current = self.inspect_service(
|
||||
service, insert_defaults=inspect_defaults
|
||||
)['Spec']
|
||||
|
||||
else:
|
||||
current = {}
|
||||
|
||||
url = self._url('/services/{0}/update', service)
|
||||
data = {}
|
||||
headers = {}
|
||||
|
||||
data['Name'] = current.get('Name') if name is None else name
|
||||
|
||||
data['Labels'] = current.get('Labels') if labels is None else labels
|
||||
|
||||
if mode is not None:
|
||||
if not isinstance(mode, dict):
|
||||
mode = ServiceMode(mode)
|
||||
data['Mode'] = mode
|
||||
else:
|
||||
data['Mode'] = current.get('Mode')
|
||||
|
||||
data['TaskTemplate'] = _merge_task_template(
|
||||
current.get('TaskTemplate', {}), task_template
|
||||
)
|
||||
|
||||
container_spec = data['TaskTemplate'].get('ContainerSpec', {})
|
||||
image = container_spec.get('Image', None)
|
||||
if image is not None:
|
||||
registry, repo_name = auth.resolve_repository_name(image)
|
||||
auth_header = auth.get_config_header(self, registry)
|
||||
if auth_header:
|
||||
headers['X-Registry-Auth'] = auth_header
|
||||
|
||||
if update_config is not None:
|
||||
data['UpdateConfig'] = update_config
|
||||
else:
|
||||
data['UpdateConfig'] = current.get('UpdateConfig')
|
||||
|
||||
if rollback_config is not None:
|
||||
data['RollbackConfig'] = rollback_config
|
||||
else:
|
||||
data['RollbackConfig'] = current.get('RollbackConfig')
|
||||
|
||||
if networks is not None:
|
||||
converted_networks = utils.convert_service_networks(networks)
|
||||
if utils.version_lt(self._version, '1.25'):
|
||||
data['Networks'] = converted_networks
|
||||
else:
|
||||
data['TaskTemplate']['Networks'] = converted_networks
|
||||
elif utils.version_lt(self._version, '1.25'):
|
||||
data['Networks'] = current.get('Networks')
|
||||
elif data['TaskTemplate'].get('Networks') is None:
|
||||
current_task_template = current.get('TaskTemplate', {})
|
||||
current_networks = current_task_template.get('Networks')
|
||||
if current_networks is None:
|
||||
current_networks = current.get('Networks')
|
||||
if current_networks is not None:
|
||||
data['TaskTemplate']['Networks'] = current_networks
|
||||
|
||||
if endpoint_spec is not None:
|
||||
data['EndpointSpec'] = endpoint_spec
|
||||
else:
|
||||
data['EndpointSpec'] = current.get('EndpointSpec')
|
||||
|
||||
resp = self._post_json(
|
||||
url, data=data, params={'version': version}, headers=headers
|
||||
)
|
||||
return self._result(resp, json=True)
|
||||
@@ -0,0 +1,462 @@
|
||||
import http.client as http_client
|
||||
import logging
|
||||
|
||||
from .. import errors, types, utils
|
||||
from ..constants import DEFAULT_SWARM_ADDR_POOL, DEFAULT_SWARM_SUBNET_SIZE
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SwarmApiMixin:
|
||||
|
||||
def create_swarm_spec(self, *args, **kwargs):
|
||||
"""
|
||||
Create a :py:class:`docker.types.SwarmSpec` instance that can be used
|
||||
as the ``swarm_spec`` argument in
|
||||
:py:meth:`~docker.api.swarm.SwarmApiMixin.init_swarm`.
|
||||
|
||||
Args:
|
||||
task_history_retention_limit (int): Maximum number of tasks
|
||||
history stored.
|
||||
snapshot_interval (int): Number of logs entries between snapshot.
|
||||
keep_old_snapshots (int): Number of snapshots to keep beyond the
|
||||
current snapshot.
|
||||
log_entries_for_slow_followers (int): Number of log entries to
|
||||
keep around to sync up slow followers after a snapshot is
|
||||
created.
|
||||
heartbeat_tick (int): Amount of ticks (in seconds) between each
|
||||
heartbeat.
|
||||
election_tick (int): Amount of ticks (in seconds) needed without a
|
||||
leader to trigger a new election.
|
||||
dispatcher_heartbeat_period (int): The delay for an agent to send
|
||||
a heartbeat to the dispatcher.
|
||||
node_cert_expiry (int): Automatic expiry for nodes certificates.
|
||||
external_cas (:py:class:`list`): Configuration for forwarding
|
||||
signing requests to an external certificate authority. Use
|
||||
a list of :py:class:`docker.types.SwarmExternalCA`.
|
||||
name (string): Swarm's name
|
||||
labels (dict): User-defined key/value metadata.
|
||||
signing_ca_cert (str): The desired signing CA certificate for all
|
||||
swarm node TLS leaf certificates, in PEM format.
|
||||
signing_ca_key (str): The desired signing CA key for all swarm
|
||||
node TLS leaf certificates, in PEM format.
|
||||
ca_force_rotate (int): An integer whose purpose is to force swarm
|
||||
to generate a new signing CA certificate and key, if none have
|
||||
been specified.
|
||||
autolock_managers (boolean): If set, generate a key and use it to
|
||||
lock data stored on the managers.
|
||||
log_driver (DriverConfig): The default log driver to use for tasks
|
||||
created in the orchestrator.
|
||||
|
||||
Returns:
|
||||
:py:class:`docker.types.SwarmSpec`
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> spec = client.api.create_swarm_spec(
|
||||
snapshot_interval=5000, log_entries_for_slow_followers=1200
|
||||
)
|
||||
>>> client.api.init_swarm(
|
||||
advertise_addr='eth0', listen_addr='0.0.0.0:5000',
|
||||
force_new_cluster=False, swarm_spec=spec
|
||||
)
|
||||
"""
|
||||
ext_ca = kwargs.pop('external_ca', None)
|
||||
if ext_ca:
|
||||
kwargs['external_cas'] = [ext_ca]
|
||||
return types.SwarmSpec(self._version, *args, **kwargs)
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
def get_unlock_key(self):
|
||||
"""
|
||||
Get the unlock key for this Swarm manager.
|
||||
|
||||
Returns:
|
||||
A ``dict`` containing an ``UnlockKey`` member
|
||||
"""
|
||||
return self._result(self._get(self._url('/swarm/unlockkey')), True)
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377',
|
||||
force_new_cluster=False, swarm_spec=None,
|
||||
default_addr_pool=None, subnet_size=None,
|
||||
data_path_addr=None, data_path_port=None):
|
||||
"""
|
||||
Initialize a new Swarm using the current connected engine as the first
|
||||
node.
|
||||
|
||||
Args:
|
||||
advertise_addr (string): Externally reachable address advertised
|
||||
to other nodes. This can either be an address/port combination
|
||||
in the form ``192.168.1.1:4567``, or an interface followed by a
|
||||
port number, like ``eth0:4567``. If the port number is omitted,
|
||||
the port number from the listen address is used. If
|
||||
``advertise_addr`` is not specified, it will be automatically
|
||||
detected when possible. Default: None
|
||||
listen_addr (string): Listen address used for inter-manager
|
||||
communication, as well as determining the networking interface
|
||||
used for the VXLAN Tunnel Endpoint (VTEP). This can either be
|
||||
an address/port combination in the form ``192.168.1.1:4567``,
|
||||
or an interface followed by a port number, like ``eth0:4567``.
|
||||
If the port number is omitted, the default swarm listening port
|
||||
is used. Default: '0.0.0.0:2377'
|
||||
force_new_cluster (bool): Force creating a new Swarm, even if
|
||||
already part of one. Default: False
|
||||
swarm_spec (dict): Configuration settings of the new Swarm. Use
|
||||
``APIClient.create_swarm_spec`` to generate a valid
|
||||
configuration. Default: None
|
||||
default_addr_pool (list of strings): Default Address Pool specifies
|
||||
default subnet pools for global scope networks. Each pool
|
||||
should be specified as a CIDR block, like '10.0.0.0/8'.
|
||||
Default: None
|
||||
subnet_size (int): SubnetSize specifies the subnet size of the
|
||||
networks created from the default subnet pool. Default: None
|
||||
data_path_addr (string): Address or interface to use for data path
|
||||
traffic. For example, 192.168.1.1, or an interface, like eth0.
|
||||
data_path_port (int): Port number to use for data path traffic.
|
||||
Acceptable port range is 1024 to 49151. If set to ``None`` or
|
||||
0, the default port 4789 will be used. Default: None
|
||||
|
||||
Returns:
|
||||
(str): The ID of the created node.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
|
||||
url = self._url('/swarm/init')
|
||||
if swarm_spec is not None and not isinstance(swarm_spec, dict):
|
||||
raise TypeError('swarm_spec must be a dictionary')
|
||||
|
||||
if default_addr_pool is not None:
|
||||
if utils.version_lt(self._version, '1.39'):
|
||||
raise errors.InvalidVersion(
|
||||
'Address pool is only available for API version >= 1.39'
|
||||
)
|
||||
# subnet_size becomes 0 if not set with default_addr_pool
|
||||
if subnet_size is None:
|
||||
subnet_size = DEFAULT_SWARM_SUBNET_SIZE
|
||||
|
||||
if subnet_size is not None:
|
||||
if utils.version_lt(self._version, '1.39'):
|
||||
raise errors.InvalidVersion(
|
||||
'Subnet size is only available for API version >= 1.39'
|
||||
)
|
||||
# subnet_size is ignored if set without default_addr_pool
|
||||
if default_addr_pool is None:
|
||||
default_addr_pool = DEFAULT_SWARM_ADDR_POOL
|
||||
|
||||
data = {
|
||||
'AdvertiseAddr': advertise_addr,
|
||||
'ListenAddr': listen_addr,
|
||||
'DefaultAddrPool': default_addr_pool,
|
||||
'SubnetSize': subnet_size,
|
||||
'ForceNewCluster': force_new_cluster,
|
||||
'Spec': swarm_spec,
|
||||
}
|
||||
|
||||
if data_path_addr is not None:
|
||||
if utils.version_lt(self._version, '1.30'):
|
||||
raise errors.InvalidVersion(
|
||||
'Data address path is only available for '
|
||||
'API version >= 1.30'
|
||||
)
|
||||
data['DataPathAddr'] = data_path_addr
|
||||
|
||||
if data_path_port is not None:
|
||||
if utils.version_lt(self._version, '1.40'):
|
||||
raise errors.InvalidVersion(
|
||||
'Data path port is only available for '
|
||||
'API version >= 1.40'
|
||||
)
|
||||
data['DataPathPort'] = data_path_port
|
||||
|
||||
response = self._post_json(url, data=data)
|
||||
return self._result(response, json=True)
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
def inspect_swarm(self):
|
||||
"""
|
||||
Retrieve low-level information about the current swarm.
|
||||
|
||||
Returns:
|
||||
A dictionary containing data about the swarm.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
url = self._url('/swarm')
|
||||
return self._result(self._get(url), True)
|
||||
|
||||
@utils.check_resource('node_id')
|
||||
@utils.minimum_version('1.24')
|
||||
def inspect_node(self, node_id):
|
||||
"""
|
||||
Retrieve low-level information about a swarm node
|
||||
|
||||
Args:
|
||||
node_id (string): ID of the node to be inspected.
|
||||
|
||||
Returns:
|
||||
A dictionary containing data about this node.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
url = self._url('/nodes/{0}', node_id)
|
||||
return self._result(self._get(url), True)
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
def join_swarm(self, remote_addrs, join_token, listen_addr='0.0.0.0:2377',
|
||||
advertise_addr=None, data_path_addr=None):
|
||||
"""
|
||||
Make this Engine join a swarm that has already been created.
|
||||
|
||||
Args:
|
||||
remote_addrs (:py:class:`list`): Addresses of one or more manager
|
||||
nodes already participating in the Swarm to join.
|
||||
join_token (string): Secret token for joining this Swarm.
|
||||
listen_addr (string): Listen address used for inter-manager
|
||||
communication if the node gets promoted to manager, as well as
|
||||
determining the networking interface used for the VXLAN Tunnel
|
||||
Endpoint (VTEP). Default: ``'0.0.0.0:2377``
|
||||
advertise_addr (string): Externally reachable address advertised
|
||||
to other nodes. This can either be an address/port combination
|
||||
in the form ``192.168.1.1:4567``, or an interface followed by a
|
||||
port number, like ``eth0:4567``. If the port number is omitted,
|
||||
the port number from the listen address is used. If
|
||||
AdvertiseAddr is not specified, it will be automatically
|
||||
detected when possible. Default: ``None``
|
||||
data_path_addr (string): Address or interface to use for data path
|
||||
traffic. For example, 192.168.1.1, or an interface, like eth0.
|
||||
|
||||
Returns:
|
||||
``True`` if the request went through.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
data = {
|
||||
'RemoteAddrs': remote_addrs,
|
||||
'ListenAddr': listen_addr,
|
||||
'JoinToken': join_token,
|
||||
'AdvertiseAddr': advertise_addr,
|
||||
}
|
||||
|
||||
if data_path_addr is not None:
|
||||
if utils.version_lt(self._version, '1.30'):
|
||||
raise errors.InvalidVersion(
|
||||
'Data address path is only available for '
|
||||
'API version >= 1.30'
|
||||
)
|
||||
data['DataPathAddr'] = data_path_addr
|
||||
|
||||
url = self._url('/swarm/join')
|
||||
response = self._post_json(url, data=data)
|
||||
self._raise_for_status(response)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
def leave_swarm(self, force=False):
|
||||
"""
|
||||
Leave a swarm.
|
||||
|
||||
Args:
|
||||
force (bool): Leave the swarm even if this node is a manager.
|
||||
Default: ``False``
|
||||
|
||||
Returns:
|
||||
``True`` if the request went through.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
url = self._url('/swarm/leave')
|
||||
response = self._post(url, params={'force': force})
|
||||
# Ignore "this node is not part of a swarm" error
|
||||
if force and response.status_code == http_client.NOT_ACCEPTABLE:
|
||||
return True
|
||||
# FIXME: Temporary workaround for 1.13.0-rc bug
|
||||
# https://github.com/docker/docker/issues/29192
|
||||
if force and response.status_code == http_client.SERVICE_UNAVAILABLE:
|
||||
return True
|
||||
self._raise_for_status(response)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
def nodes(self, filters=None):
|
||||
"""
|
||||
List swarm nodes.
|
||||
|
||||
Args:
|
||||
filters (dict): Filters to process on the nodes list. Valid
|
||||
filters: ``id``, ``name``, ``membership`` and ``role``.
|
||||
Default: ``None``
|
||||
|
||||
Returns:
|
||||
A list of dictionaries containing data about each swarm node.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
url = self._url('/nodes')
|
||||
params = {}
|
||||
if filters:
|
||||
params['filters'] = utils.convert_filters(filters)
|
||||
|
||||
return self._result(self._get(url, params=params), True)
|
||||
|
||||
@utils.check_resource('node_id')
|
||||
@utils.minimum_version('1.24')
|
||||
def remove_node(self, node_id, force=False):
|
||||
"""
|
||||
Remove a node from the swarm.
|
||||
|
||||
Args:
|
||||
node_id (string): ID of the node to be removed.
|
||||
force (bool): Force remove an active node. Default: `False`
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.NotFound`
|
||||
If the node referenced doesn't exist in the swarm.
|
||||
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
Returns:
|
||||
`True` if the request was successful.
|
||||
"""
|
||||
url = self._url('/nodes/{0}', node_id)
|
||||
params = {
|
||||
'force': force
|
||||
}
|
||||
res = self._delete(url, params=params)
|
||||
self._raise_for_status(res)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
def unlock_swarm(self, key):
|
||||
"""
|
||||
Unlock a locked swarm.
|
||||
|
||||
Args:
|
||||
key (string): The unlock key as provided by
|
||||
:py:meth:`get_unlock_key`
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.InvalidArgument`
|
||||
If the key argument is in an incompatible format
|
||||
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Returns:
|
||||
`True` if the request was successful.
|
||||
|
||||
Example:
|
||||
|
||||
>>> key = client.api.get_unlock_key()
|
||||
>>> client.unlock_swarm(key)
|
||||
|
||||
"""
|
||||
if isinstance(key, dict):
|
||||
if 'UnlockKey' not in key:
|
||||
raise errors.InvalidArgument('Invalid unlock key format')
|
||||
else:
|
||||
key = {'UnlockKey': key}
|
||||
|
||||
url = self._url('/swarm/unlock')
|
||||
res = self._post_json(url, data=key)
|
||||
self._raise_for_status(res)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
def update_node(self, node_id, version, node_spec=None):
|
||||
"""
|
||||
Update the node's configuration
|
||||
|
||||
Args:
|
||||
|
||||
node_id (string): ID of the node to be updated.
|
||||
version (int): The version number of the node object being
|
||||
updated. This is required to avoid conflicting writes.
|
||||
node_spec (dict): Configuration settings to update. Any values
|
||||
not provided will be removed. Default: ``None``
|
||||
|
||||
Returns:
|
||||
`True` if the request went through.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> node_spec = {'Availability': 'active',
|
||||
'Name': 'node-name',
|
||||
'Role': 'manager',
|
||||
'Labels': {'foo': 'bar'}
|
||||
}
|
||||
>>> client.api.update_node(node_id='24ifsmvkjbyhk', version=8,
|
||||
node_spec=node_spec)
|
||||
|
||||
"""
|
||||
url = self._url('/nodes/{0}/update?version={1}', node_id, str(version))
|
||||
res = self._post_json(url, data=node_spec)
|
||||
self._raise_for_status(res)
|
||||
return True
|
||||
|
||||
@utils.minimum_version('1.24')
|
||||
def update_swarm(self, version, swarm_spec=None,
|
||||
rotate_worker_token=False,
|
||||
rotate_manager_token=False,
|
||||
rotate_manager_unlock_key=False):
|
||||
"""
|
||||
Update the Swarm's configuration
|
||||
|
||||
Args:
|
||||
version (int): The version number of the swarm object being
|
||||
updated. This is required to avoid conflicting writes.
|
||||
swarm_spec (dict): Configuration settings to update. Use
|
||||
:py:meth:`~docker.api.swarm.SwarmApiMixin.create_swarm_spec` to
|
||||
generate a valid configuration. Default: ``None``.
|
||||
rotate_worker_token (bool): Rotate the worker join token. Default:
|
||||
``False``.
|
||||
rotate_manager_token (bool): Rotate the manager join token.
|
||||
Default: ``False``.
|
||||
rotate_manager_unlock_key (bool): Rotate the manager unlock key.
|
||||
Default: ``False``.
|
||||
|
||||
Returns:
|
||||
``True`` if the request went through.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
url = self._url('/swarm/update')
|
||||
params = {
|
||||
'rotateWorkerToken': rotate_worker_token,
|
||||
'rotateManagerToken': rotate_manager_token,
|
||||
'version': version
|
||||
}
|
||||
if rotate_manager_unlock_key:
|
||||
if utils.version_lt(self._version, '1.25'):
|
||||
raise errors.InvalidVersion(
|
||||
'Rotate manager unlock key '
|
||||
'is only available for API version >= 1.25'
|
||||
)
|
||||
params['rotateManagerUnlockKey'] = rotate_manager_unlock_key
|
||||
|
||||
response = self._post_json(url, data=swarm_spec, params=params)
|
||||
self._raise_for_status(response)
|
||||
return True
|
||||
@@ -0,0 +1,163 @@
|
||||
from .. import errors, utils
|
||||
|
||||
|
||||
class VolumeApiMixin:
|
||||
def volumes(self, filters=None):
|
||||
"""
|
||||
List volumes currently registered by the docker daemon. Similar to the
|
||||
``docker volume ls`` command.
|
||||
|
||||
Args:
|
||||
filters (dict): Server-side list filtering options.
|
||||
|
||||
Returns:
|
||||
(dict): Dictionary with list of volume objects as value of the
|
||||
``Volumes`` key.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> client.api.volumes()
|
||||
{u'Volumes': [{u'Driver': u'local',
|
||||
u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data',
|
||||
u'Name': u'foobar'},
|
||||
{u'Driver': u'local',
|
||||
u'Mountpoint': u'/var/lib/docker/volumes/baz/_data',
|
||||
u'Name': u'baz'}]}
|
||||
"""
|
||||
|
||||
params = {
|
||||
'filters': utils.convert_filters(filters) if filters else None
|
||||
}
|
||||
url = self._url('/volumes')
|
||||
return self._result(self._get(url, params=params), True)
|
||||
|
||||
def create_volume(self, name=None, driver=None, driver_opts=None,
|
||||
labels=None):
|
||||
"""
|
||||
Create and register a named volume
|
||||
|
||||
Args:
|
||||
name (str): Name of the volume
|
||||
driver (str): Name of the driver used to create the volume
|
||||
driver_opts (dict): Driver options as a key-value dictionary
|
||||
labels (dict): Labels to set on the volume
|
||||
|
||||
Returns:
|
||||
(dict): The created volume reference object
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> volume = client.api.create_volume(
|
||||
... name='foobar',
|
||||
... driver='local',
|
||||
... driver_opts={'foo': 'bar', 'baz': 'false'},
|
||||
... labels={"key": "value"},
|
||||
... )
|
||||
... print(volume)
|
||||
{u'Driver': u'local',
|
||||
u'Labels': {u'key': u'value'},
|
||||
u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data',
|
||||
u'Name': u'foobar',
|
||||
u'Scope': u'local'}
|
||||
|
||||
"""
|
||||
url = self._url('/volumes/create')
|
||||
if driver_opts is not None and not isinstance(driver_opts, dict):
|
||||
raise TypeError('driver_opts must be a dictionary')
|
||||
|
||||
data = {
|
||||
'Name': name,
|
||||
'Driver': driver,
|
||||
'DriverOpts': driver_opts,
|
||||
}
|
||||
|
||||
if labels is not None:
|
||||
if utils.compare_version('1.23', self._version) < 0:
|
||||
raise errors.InvalidVersion(
|
||||
'volume labels were introduced in API 1.23'
|
||||
)
|
||||
if not isinstance(labels, dict):
|
||||
raise TypeError('labels must be a dictionary')
|
||||
data["Labels"] = labels
|
||||
|
||||
return self._result(self._post_json(url, data=data), True)
|
||||
|
||||
def inspect_volume(self, name):
|
||||
"""
|
||||
Retrieve volume info by name.
|
||||
|
||||
Args:
|
||||
name (str): volume name
|
||||
|
||||
Returns:
|
||||
(dict): Volume information dictionary
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> client.api.inspect_volume('foobar')
|
||||
{u'Driver': u'local',
|
||||
u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data',
|
||||
u'Name': u'foobar'}
|
||||
|
||||
"""
|
||||
url = self._url('/volumes/{0}', name)
|
||||
return self._result(self._get(url), True)
|
||||
|
||||
@utils.minimum_version('1.25')
|
||||
def prune_volumes(self, filters=None):
|
||||
"""
|
||||
Delete unused volumes
|
||||
|
||||
Args:
|
||||
filters (dict): Filters to process on the prune list.
|
||||
|
||||
Returns:
|
||||
(dict): A dict containing a list of deleted volume names and
|
||||
the amount of disk space reclaimed in bytes.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
params = {}
|
||||
if filters:
|
||||
params['filters'] = utils.convert_filters(filters)
|
||||
url = self._url('/volumes/prune')
|
||||
return self._result(self._post(url, params=params), True)
|
||||
|
||||
def remove_volume(self, name, force=False):
|
||||
"""
|
||||
Remove a volume. Similar to the ``docker volume rm`` command.
|
||||
|
||||
Args:
|
||||
name (str): The volume's name
|
||||
force (bool): Force removal of volumes that were already removed
|
||||
out of band by the volume driver plugin.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If volume failed to remove.
|
||||
"""
|
||||
params = {}
|
||||
if force:
|
||||
if utils.version_lt(self._version, '1.25'):
|
||||
raise errors.InvalidVersion(
|
||||
'force removal was introduced in API 1.25'
|
||||
)
|
||||
params = {'force': force}
|
||||
|
||||
url = self._url('/volumes/{0}', name, params=params)
|
||||
resp = self._delete(url)
|
||||
self._raise_for_status(resp)
|
||||
@@ -0,0 +1,378 @@
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
|
||||
from . import credentials, errors
|
||||
from .utils import config
|
||||
|
||||
INDEX_NAME = 'docker.io'
|
||||
INDEX_URL = f'https://index.{INDEX_NAME}/v1/'
|
||||
TOKEN_USERNAME = '<token>'
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def resolve_repository_name(repo_name):
|
||||
if '://' in repo_name:
|
||||
raise errors.InvalidRepository(
|
||||
f'Repository name cannot contain a scheme ({repo_name})'
|
||||
)
|
||||
|
||||
index_name, remote_name = split_repo_name(repo_name)
|
||||
if index_name[0] == '-' or index_name[-1] == '-':
|
||||
raise errors.InvalidRepository(
|
||||
f'Invalid index name ({index_name}). '
|
||||
'Cannot begin or end with a hyphen.'
|
||||
)
|
||||
return resolve_index_name(index_name), remote_name
|
||||
|
||||
|
||||
def resolve_index_name(index_name):
|
||||
index_name = convert_to_hostname(index_name)
|
||||
if index_name == f"index.{INDEX_NAME}":
|
||||
index_name = INDEX_NAME
|
||||
return index_name
|
||||
|
||||
|
||||
def get_config_header(client, registry):
|
||||
log.debug('Looking for auth config')
|
||||
if not client._auth_configs or client._auth_configs.is_empty:
|
||||
log.debug(
|
||||
"No auth config in memory - loading from filesystem"
|
||||
)
|
||||
client._auth_configs = load_config(credstore_env=client.credstore_env)
|
||||
authcfg = resolve_authconfig(
|
||||
client._auth_configs, registry, credstore_env=client.credstore_env
|
||||
)
|
||||
# Do not fail here if no authentication exists for this
|
||||
# specific registry as we can have a readonly pull. Just
|
||||
# put the header if we can.
|
||||
if authcfg:
|
||||
log.debug('Found auth config')
|
||||
# auth_config needs to be a dict in the format used by
|
||||
# auth.py username , password, serveraddress, email
|
||||
return encode_header(authcfg)
|
||||
log.debug('No auth config found')
|
||||
return None
|
||||
|
||||
|
||||
def split_repo_name(repo_name):
|
||||
parts = repo_name.split('/', 1)
|
||||
if len(parts) == 1 or (
|
||||
'.' not in parts[0] and ':' not in parts[0] and parts[0] != 'localhost'
|
||||
):
|
||||
# This is a docker index repo (ex: username/foobar or ubuntu)
|
||||
return INDEX_NAME, repo_name
|
||||
return tuple(parts)
|
||||
|
||||
|
||||
def get_credential_store(authconfig, registry):
|
||||
if not isinstance(authconfig, AuthConfig):
|
||||
authconfig = AuthConfig(authconfig)
|
||||
return authconfig.get_credential_store(registry)
|
||||
|
||||
|
||||
class AuthConfig(dict):
|
||||
def __init__(self, dct, credstore_env=None):
|
||||
if 'auths' not in dct:
|
||||
dct['auths'] = {}
|
||||
self.update(dct)
|
||||
self._credstore_env = credstore_env
|
||||
self._stores = {}
|
||||
|
||||
@classmethod
|
||||
def parse_auth(cls, entries, raise_on_error=False):
|
||||
"""
|
||||
Parses authentication entries
|
||||
|
||||
Args:
|
||||
entries: Dict of authentication entries.
|
||||
raise_on_error: If set to true, an invalid format will raise
|
||||
InvalidConfigFile
|
||||
|
||||
Returns:
|
||||
Authentication registry.
|
||||
"""
|
||||
|
||||
conf = {}
|
||||
for registry, entry in entries.items():
|
||||
if not isinstance(entry, dict):
|
||||
log.debug(
|
||||
f'Config entry for key {registry} is not auth config'
|
||||
)
|
||||
# We sometimes fall back to parsing the whole config as if it
|
||||
# was the auth config by itself, for legacy purposes. In that
|
||||
# case, we fail silently and return an empty conf if any of the
|
||||
# keys is not formatted properly.
|
||||
if raise_on_error:
|
||||
raise errors.InvalidConfigFile(
|
||||
f'Invalid configuration for registry {registry}'
|
||||
)
|
||||
return {}
|
||||
if 'identitytoken' in entry:
|
||||
log.debug(f'Found an IdentityToken entry for registry {registry}')
|
||||
conf[registry] = {
|
||||
'IdentityToken': entry['identitytoken']
|
||||
}
|
||||
continue # Other values are irrelevant if we have a token
|
||||
|
||||
if 'auth' not in entry:
|
||||
# Starting with engine v1.11 (API 1.23), an empty dictionary is
|
||||
# a valid value in the auths config.
|
||||
# https://github.com/docker/compose/issues/3265
|
||||
log.debug(
|
||||
f'Auth data for {registry} is absent. '
|
||||
f'Client might be using a credentials store instead.'
|
||||
)
|
||||
conf[registry] = {}
|
||||
continue
|
||||
|
||||
username, password = decode_auth(entry['auth'])
|
||||
log.debug(
|
||||
f'Found entry (registry={registry!r}, username={username!r})'
|
||||
)
|
||||
|
||||
conf[registry] = {
|
||||
'username': username,
|
||||
'password': password,
|
||||
'email': entry.get('email'),
|
||||
'serveraddress': registry,
|
||||
}
|
||||
return conf
|
||||
|
||||
@classmethod
|
||||
def load_config(cls, config_path, config_dict, credstore_env=None):
|
||||
"""
|
||||
Loads authentication data from a Docker configuration file in the given
|
||||
root directory or if config_path is passed use given path.
|
||||
Lookup priority:
|
||||
explicit config_path parameter > DOCKER_CONFIG environment
|
||||
variable > ~/.docker/config.json > ~/.dockercfg
|
||||
"""
|
||||
|
||||
if not config_dict:
|
||||
config_file = config.find_config_file(config_path)
|
||||
|
||||
if not config_file:
|
||||
return cls({}, credstore_env)
|
||||
try:
|
||||
with open(config_file) as f:
|
||||
config_dict = json.load(f)
|
||||
except (OSError, KeyError, ValueError) as e:
|
||||
# Likely missing new Docker config file or it's in an
|
||||
# unknown format, continue to attempt to read old location
|
||||
# and format.
|
||||
log.debug(e)
|
||||
return cls(_load_legacy_config(config_file), credstore_env)
|
||||
|
||||
res = {}
|
||||
if config_dict.get('auths'):
|
||||
log.debug("Found 'auths' section")
|
||||
res.update({
|
||||
'auths': cls.parse_auth(
|
||||
config_dict.pop('auths'), raise_on_error=True
|
||||
)
|
||||
})
|
||||
if config_dict.get('credsStore'):
|
||||
log.debug("Found 'credsStore' section")
|
||||
res.update({'credsStore': config_dict.pop('credsStore')})
|
||||
if config_dict.get('credHelpers'):
|
||||
log.debug("Found 'credHelpers' section")
|
||||
res.update({'credHelpers': config_dict.pop('credHelpers')})
|
||||
if res:
|
||||
return cls(res, credstore_env)
|
||||
|
||||
log.debug(
|
||||
"Couldn't find auth-related section ; attempting to interpret "
|
||||
"as auth-only file"
|
||||
)
|
||||
return cls({'auths': cls.parse_auth(config_dict)}, credstore_env)
|
||||
|
||||
@property
|
||||
def auths(self):
|
||||
return self.get('auths', {})
|
||||
|
||||
@property
|
||||
def creds_store(self):
|
||||
return self.get('credsStore', None)
|
||||
|
||||
@property
|
||||
def cred_helpers(self):
|
||||
return self.get('credHelpers', {})
|
||||
|
||||
@property
|
||||
def is_empty(self):
|
||||
return (
|
||||
not self.auths and not self.creds_store and not self.cred_helpers
|
||||
)
|
||||
|
||||
def resolve_authconfig(self, registry=None):
|
||||
"""
|
||||
Returns the authentication data from the given auth configuration for a
|
||||
specific registry. As with the Docker client, legacy entries in the
|
||||
config with full URLs are stripped down to hostnames before checking
|
||||
for a match. Returns None if no match was found.
|
||||
"""
|
||||
|
||||
if self.creds_store or self.cred_helpers:
|
||||
store_name = self.get_credential_store(registry)
|
||||
if store_name is not None:
|
||||
log.debug(
|
||||
f'Using credentials store "{store_name}"'
|
||||
)
|
||||
cfg = self._resolve_authconfig_credstore(registry, store_name)
|
||||
if cfg is not None:
|
||||
return cfg
|
||||
log.debug('No entry in credstore - fetching from auth dict')
|
||||
|
||||
# Default to the public index server
|
||||
registry = resolve_index_name(registry) if registry else INDEX_NAME
|
||||
log.debug(f"Looking for auth entry for {repr(registry)}")
|
||||
|
||||
if registry in self.auths:
|
||||
log.debug(f"Found {repr(registry)}")
|
||||
return self.auths[registry]
|
||||
|
||||
for key, conf in self.auths.items():
|
||||
if resolve_index_name(key) == registry:
|
||||
log.debug(f"Found {repr(key)}")
|
||||
return conf
|
||||
|
||||
log.debug("No entry found")
|
||||
return None
|
||||
|
||||
def _resolve_authconfig_credstore(self, registry, credstore_name):
|
||||
if not registry or registry == INDEX_NAME:
|
||||
# The ecosystem is a little schizophrenic with index.docker.io VS
|
||||
# docker.io - in that case, it seems the full URL is necessary.
|
||||
registry = INDEX_URL
|
||||
log.debug(f"Looking for auth entry for {repr(registry)}")
|
||||
store = self._get_store_instance(credstore_name)
|
||||
try:
|
||||
data = store.get(registry)
|
||||
res = {
|
||||
'ServerAddress': registry,
|
||||
}
|
||||
if data['Username'] == TOKEN_USERNAME:
|
||||
res['IdentityToken'] = data['Secret']
|
||||
else:
|
||||
res.update({
|
||||
'Username': data['Username'],
|
||||
'Password': data['Secret'],
|
||||
})
|
||||
return res
|
||||
except credentials.CredentialsNotFound:
|
||||
log.debug('No entry found')
|
||||
return None
|
||||
except credentials.StoreError as e:
|
||||
raise errors.DockerException(
|
||||
f'Credentials store error: {repr(e)}'
|
||||
) from e
|
||||
|
||||
def _get_store_instance(self, name):
|
||||
if name not in self._stores:
|
||||
self._stores[name] = credentials.Store(
|
||||
name, environment=self._credstore_env
|
||||
)
|
||||
return self._stores[name]
|
||||
|
||||
def get_credential_store(self, registry):
|
||||
if not registry or registry == INDEX_NAME:
|
||||
registry = INDEX_URL
|
||||
|
||||
return self.cred_helpers.get(registry) or self.creds_store
|
||||
|
||||
def get_all_credentials(self):
|
||||
auth_data = self.auths.copy()
|
||||
if self.creds_store:
|
||||
# Retrieve all credentials from the default store
|
||||
store = self._get_store_instance(self.creds_store)
|
||||
for k in store.list().keys():
|
||||
auth_data[k] = self._resolve_authconfig_credstore(
|
||||
k, self.creds_store
|
||||
)
|
||||
auth_data[convert_to_hostname(k)] = auth_data[k]
|
||||
|
||||
# credHelpers entries take priority over all others
|
||||
for reg, store_name in self.cred_helpers.items():
|
||||
auth_data[reg] = self._resolve_authconfig_credstore(
|
||||
reg, store_name
|
||||
)
|
||||
auth_data[convert_to_hostname(reg)] = auth_data[reg]
|
||||
|
||||
return auth_data
|
||||
|
||||
def add_auth(self, reg, data):
|
||||
self['auths'][reg] = data
|
||||
|
||||
|
||||
def resolve_authconfig(authconfig, registry=None, credstore_env=None):
|
||||
if not isinstance(authconfig, AuthConfig):
|
||||
authconfig = AuthConfig(authconfig, credstore_env)
|
||||
return authconfig.resolve_authconfig(registry)
|
||||
|
||||
|
||||
def convert_to_hostname(url):
|
||||
return url.replace('http://', '').replace('https://', '').split('/', 1)[0]
|
||||
|
||||
|
||||
def decode_auth(auth):
|
||||
if isinstance(auth, str):
|
||||
auth = auth.encode('ascii')
|
||||
s = base64.b64decode(auth)
|
||||
login, pwd = s.split(b':', 1)
|
||||
return login.decode('utf8'), pwd.decode('utf8')
|
||||
|
||||
|
||||
def encode_header(auth):
|
||||
auth_json = json.dumps(auth).encode('ascii')
|
||||
return base64.urlsafe_b64encode(auth_json)
|
||||
|
||||
|
||||
def parse_auth(entries, raise_on_error=False):
|
||||
"""
|
||||
Parses authentication entries
|
||||
|
||||
Args:
|
||||
entries: Dict of authentication entries.
|
||||
raise_on_error: If set to true, an invalid format will raise
|
||||
InvalidConfigFile
|
||||
|
||||
Returns:
|
||||
Authentication registry.
|
||||
"""
|
||||
|
||||
return AuthConfig.parse_auth(entries, raise_on_error)
|
||||
|
||||
|
||||
def load_config(config_path=None, config_dict=None, credstore_env=None):
|
||||
return AuthConfig.load_config(config_path, config_dict, credstore_env)
|
||||
|
||||
|
||||
def _load_legacy_config(config_file):
|
||||
log.debug("Attempting to parse legacy auth file format")
|
||||
try:
|
||||
data = []
|
||||
with open(config_file) as f:
|
||||
for line in f.readlines():
|
||||
data.append(line.strip().split(' = ')[1])
|
||||
if len(data) < 2:
|
||||
# Not enough data
|
||||
raise errors.InvalidConfigFile(
|
||||
'Invalid or empty configuration file!'
|
||||
)
|
||||
|
||||
username, password = decode_auth(data[0])
|
||||
return {'auths': {
|
||||
INDEX_NAME: {
|
||||
'username': username,
|
||||
'password': password,
|
||||
'email': data[1],
|
||||
'serveraddress': INDEX_URL,
|
||||
}
|
||||
}}
|
||||
except Exception as e:
|
||||
log.debug(e)
|
||||
|
||||
log.debug("All parsing attempts failed - returning empty config")
|
||||
return {}
|
||||
@@ -0,0 +1,222 @@
|
||||
from .api.client import APIClient
|
||||
from .constants import DEFAULT_MAX_POOL_SIZE, DEFAULT_TIMEOUT_SECONDS
|
||||
from .models.configs import ConfigCollection
|
||||
from .models.containers import ContainerCollection
|
||||
from .models.images import ImageCollection
|
||||
from .models.networks import NetworkCollection
|
||||
from .models.nodes import NodeCollection
|
||||
from .models.plugins import PluginCollection
|
||||
from .models.secrets import SecretCollection
|
||||
from .models.services import ServiceCollection
|
||||
from .models.swarm import Swarm
|
||||
from .models.volumes import VolumeCollection
|
||||
from .utils import kwargs_from_env
|
||||
|
||||
|
||||
class DockerClient:
|
||||
"""
|
||||
A client for communicating with a Docker server.
|
||||
|
||||
Example:
|
||||
|
||||
>>> import docker
|
||||
>>> client = docker.DockerClient(base_url='unix://var/run/docker.sock')
|
||||
|
||||
Args:
|
||||
base_url (str): URL to the Docker server. For example,
|
||||
``unix:///var/run/docker.sock`` or ``tcp://127.0.0.1:1234``.
|
||||
version (str): The version of the API to use. Set to ``auto`` to
|
||||
automatically detect the server's version. Default: ``1.35``
|
||||
timeout (int): Default timeout for API calls, in seconds.
|
||||
tls (bool or :py:class:`~docker.tls.TLSConfig`): Enable TLS. Pass
|
||||
``True`` to enable it with default options, or pass a
|
||||
:py:class:`~docker.tls.TLSConfig` object to use custom
|
||||
configuration.
|
||||
user_agent (str): Set a custom user agent for requests to the server.
|
||||
credstore_env (dict): Override environment variables when calling the
|
||||
credential store process.
|
||||
use_ssh_client (bool): If set to `True`, an ssh connection is made
|
||||
via shelling out to the ssh client. Ensure the ssh client is
|
||||
installed and configured on the host.
|
||||
max_pool_size (int): The maximum number of connections
|
||||
to save in the pool.
|
||||
"""
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.api = APIClient(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, **kwargs):
|
||||
"""
|
||||
Return a client configured from environment variables.
|
||||
|
||||
The environment variables used are the same as those used by the
|
||||
Docker command-line client. They are:
|
||||
|
||||
.. envvar:: DOCKER_HOST
|
||||
|
||||
The URL to the Docker host.
|
||||
|
||||
.. envvar:: DOCKER_TLS_VERIFY
|
||||
|
||||
Verify the host against a CA certificate.
|
||||
|
||||
.. envvar:: DOCKER_CERT_PATH
|
||||
|
||||
A path to a directory containing TLS certificates to use when
|
||||
connecting to the Docker host.
|
||||
|
||||
Args:
|
||||
version (str): The version of the API to use. Set to ``auto`` to
|
||||
automatically detect the server's version. Default: ``auto``
|
||||
timeout (int): Default timeout for API calls, in seconds.
|
||||
max_pool_size (int): The maximum number of connections
|
||||
to save in the pool.
|
||||
environment (dict): The environment to read environment variables
|
||||
from. Default: the value of ``os.environ``
|
||||
credstore_env (dict): Override environment variables when calling
|
||||
the credential store process.
|
||||
use_ssh_client (bool): If set to `True`, an ssh connection is
|
||||
made via shelling out to the ssh client. Ensure the ssh
|
||||
client is installed and configured on the host.
|
||||
|
||||
Example:
|
||||
|
||||
>>> import docker
|
||||
>>> client = docker.from_env()
|
||||
|
||||
.. _`SSL version`:
|
||||
https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1
|
||||
"""
|
||||
timeout = kwargs.pop('timeout', DEFAULT_TIMEOUT_SECONDS)
|
||||
max_pool_size = kwargs.pop('max_pool_size', DEFAULT_MAX_POOL_SIZE)
|
||||
version = kwargs.pop('version', None)
|
||||
use_ssh_client = kwargs.pop('use_ssh_client', False)
|
||||
return cls(
|
||||
timeout=timeout,
|
||||
max_pool_size=max_pool_size,
|
||||
version=version,
|
||||
use_ssh_client=use_ssh_client,
|
||||
**kwargs_from_env(**kwargs)
|
||||
)
|
||||
|
||||
# Resources
|
||||
@property
|
||||
def configs(self):
|
||||
"""
|
||||
An object for managing configs on the server. See the
|
||||
:doc:`configs documentation <configs>` for full details.
|
||||
"""
|
||||
return ConfigCollection(client=self)
|
||||
|
||||
@property
|
||||
def containers(self):
|
||||
"""
|
||||
An object for managing containers on the server. See the
|
||||
:doc:`containers documentation <containers>` for full details.
|
||||
"""
|
||||
return ContainerCollection(client=self)
|
||||
|
||||
@property
|
||||
def images(self):
|
||||
"""
|
||||
An object for managing images on the server. See the
|
||||
:doc:`images documentation <images>` for full details.
|
||||
"""
|
||||
return ImageCollection(client=self)
|
||||
|
||||
@property
|
||||
def networks(self):
|
||||
"""
|
||||
An object for managing networks on the server. See the
|
||||
:doc:`networks documentation <networks>` for full details.
|
||||
"""
|
||||
return NetworkCollection(client=self)
|
||||
|
||||
@property
|
||||
def nodes(self):
|
||||
"""
|
||||
An object for managing nodes on the server. See the
|
||||
:doc:`nodes documentation <nodes>` for full details.
|
||||
"""
|
||||
return NodeCollection(client=self)
|
||||
|
||||
@property
|
||||
def plugins(self):
|
||||
"""
|
||||
An object for managing plugins on the server. See the
|
||||
:doc:`plugins documentation <plugins>` for full details.
|
||||
"""
|
||||
return PluginCollection(client=self)
|
||||
|
||||
@property
|
||||
def secrets(self):
|
||||
"""
|
||||
An object for managing secrets on the server. See the
|
||||
:doc:`secrets documentation <secrets>` for full details.
|
||||
"""
|
||||
return SecretCollection(client=self)
|
||||
|
||||
@property
|
||||
def services(self):
|
||||
"""
|
||||
An object for managing services on the server. See the
|
||||
:doc:`services documentation <services>` for full details.
|
||||
"""
|
||||
return ServiceCollection(client=self)
|
||||
|
||||
@property
|
||||
def swarm(self):
|
||||
"""
|
||||
An object for managing a swarm on the server. See the
|
||||
:doc:`swarm documentation <swarm>` for full details.
|
||||
"""
|
||||
return Swarm(client=self)
|
||||
|
||||
@property
|
||||
def volumes(self):
|
||||
"""
|
||||
An object for managing volumes on the server. See the
|
||||
:doc:`volumes documentation <volumes>` for full details.
|
||||
"""
|
||||
return VolumeCollection(client=self)
|
||||
|
||||
# Top-level methods
|
||||
def events(self, *args, **kwargs):
|
||||
return self.api.events(*args, **kwargs)
|
||||
events.__doc__ = APIClient.events.__doc__
|
||||
|
||||
def df(self):
|
||||
return self.api.df()
|
||||
df.__doc__ = APIClient.df.__doc__
|
||||
|
||||
def info(self, *args, **kwargs):
|
||||
return self.api.info(*args, **kwargs)
|
||||
info.__doc__ = APIClient.info.__doc__
|
||||
|
||||
def login(self, *args, **kwargs):
|
||||
return self.api.login(*args, **kwargs)
|
||||
login.__doc__ = APIClient.login.__doc__
|
||||
|
||||
def ping(self, *args, **kwargs):
|
||||
return self.api.ping(*args, **kwargs)
|
||||
ping.__doc__ = APIClient.ping.__doc__
|
||||
|
||||
def version(self, *args, **kwargs):
|
||||
return self.api.version(*args, **kwargs)
|
||||
version.__doc__ = APIClient.version.__doc__
|
||||
|
||||
def close(self):
|
||||
return self.api.close()
|
||||
close.__doc__ = APIClient.close.__doc__
|
||||
|
||||
def __getattr__(self, name):
|
||||
s = [f"'DockerClient' object has no attribute '{name}'"]
|
||||
# If a user calls a method on APIClient, they
|
||||
if hasattr(APIClient, name):
|
||||
s.append("In Docker SDK for Python 2.0, this method is now on the "
|
||||
"object APIClient. See the low-level API section of the "
|
||||
"documentation for more details.")
|
||||
raise AttributeError(' '.join(s))
|
||||
|
||||
|
||||
from_env = DockerClient.from_env
|
||||
@@ -0,0 +1,45 @@
|
||||
import sys
|
||||
|
||||
from .version import __version__
|
||||
|
||||
DEFAULT_DOCKER_API_VERSION = '1.44'
|
||||
MINIMUM_DOCKER_API_VERSION = '1.24'
|
||||
DEFAULT_TIMEOUT_SECONDS = 60
|
||||
STREAM_HEADER_SIZE_BYTES = 8
|
||||
CONTAINER_LIMITS_KEYS = [
|
||||
'memory', 'memswap', 'cpushares', 'cpusetcpus'
|
||||
]
|
||||
|
||||
DEFAULT_HTTP_HOST = "127.0.0.1"
|
||||
DEFAULT_UNIX_SOCKET = "http+unix:///var/run/docker.sock"
|
||||
DEFAULT_NPIPE = 'npipe:////./pipe/docker_engine'
|
||||
|
||||
BYTE_UNITS = {
|
||||
'b': 1,
|
||||
'k': 1024,
|
||||
'm': 1024 * 1024,
|
||||
'g': 1024 * 1024 * 1024
|
||||
}
|
||||
|
||||
|
||||
INSECURE_REGISTRY_DEPRECATION_WARNING = \
|
||||
'The `insecure_registry` argument to {} ' \
|
||||
'is deprecated and non-functional. Please remove it.'
|
||||
|
||||
IS_WINDOWS_PLATFORM = (sys.platform == 'win32')
|
||||
WINDOWS_LONGPATH_PREFIX = '\\\\?\\'
|
||||
|
||||
DEFAULT_USER_AGENT = f"docker-sdk-python/{__version__}"
|
||||
DEFAULT_NUM_POOLS = 25
|
||||
|
||||
# The OpenSSH server default value for MaxSessions is 10 which means we can
|
||||
# use up to 9, leaving the final session for the underlying SSH connection.
|
||||
# For more details see: https://github.com/docker/docker-py/issues/2246
|
||||
DEFAULT_NUM_POOLS_SSH = 9
|
||||
|
||||
DEFAULT_MAX_POOL_SIZE = 10
|
||||
|
||||
DEFAULT_DATA_CHUNK_SIZE = 1024 * 2048
|
||||
|
||||
DEFAULT_SWARM_ADDR_POOL = ['10.0.0.0/8']
|
||||
DEFAULT_SWARM_SUBNET_SIZE = 24
|
||||
@@ -0,0 +1,2 @@
|
||||
from .api import ContextAPI
|
||||
from .context import Context
|
||||
@@ -0,0 +1,206 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
from docker import errors
|
||||
|
||||
from .config import (
|
||||
METAFILE,
|
||||
get_current_context_name,
|
||||
get_meta_dir,
|
||||
write_context_name_to_docker_config,
|
||||
)
|
||||
from .context import Context
|
||||
|
||||
|
||||
class ContextAPI:
|
||||
"""Context API.
|
||||
Contains methods for context management:
|
||||
create, list, remove, get, inspect.
|
||||
"""
|
||||
DEFAULT_CONTEXT = Context("default", "swarm")
|
||||
|
||||
@classmethod
|
||||
def create_context(
|
||||
cls, name, orchestrator=None, host=None, tls_cfg=None,
|
||||
default_namespace=None, skip_tls_verify=False):
|
||||
"""Creates a new context.
|
||||
Returns:
|
||||
(Context): a Context object.
|
||||
Raises:
|
||||
:py:class:`docker.errors.MissingContextParameter`
|
||||
If a context name is not provided.
|
||||
:py:class:`docker.errors.ContextAlreadyExists`
|
||||
If a context with the name already exists.
|
||||
:py:class:`docker.errors.ContextException`
|
||||
If name is default.
|
||||
|
||||
Example:
|
||||
|
||||
>>> from docker.context import ContextAPI
|
||||
>>> ctx = ContextAPI.create_context(name='test')
|
||||
>>> print(ctx.Metadata)
|
||||
{
|
||||
"Name": "test",
|
||||
"Metadata": {},
|
||||
"Endpoints": {
|
||||
"docker": {
|
||||
"Host": "unix:///var/run/docker.sock",
|
||||
"SkipTLSVerify": false
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
if not name:
|
||||
raise errors.MissingContextParameter("name")
|
||||
if name == "default":
|
||||
raise errors.ContextException(
|
||||
'"default" is a reserved context name')
|
||||
ctx = Context.load_context(name)
|
||||
if ctx:
|
||||
raise errors.ContextAlreadyExists(name)
|
||||
endpoint = "docker"
|
||||
if orchestrator and orchestrator != "swarm":
|
||||
endpoint = orchestrator
|
||||
ctx = Context(name, orchestrator)
|
||||
ctx.set_endpoint(
|
||||
endpoint, host, tls_cfg,
|
||||
skip_tls_verify=skip_tls_verify,
|
||||
def_namespace=default_namespace)
|
||||
ctx.save()
|
||||
return ctx
|
||||
|
||||
@classmethod
|
||||
def get_context(cls, name=None):
|
||||
"""Retrieves a context object.
|
||||
Args:
|
||||
name (str): The name of the context
|
||||
|
||||
Example:
|
||||
|
||||
>>> from docker.context import ContextAPI
|
||||
>>> ctx = ContextAPI.get_context(name='test')
|
||||
>>> print(ctx.Metadata)
|
||||
{
|
||||
"Name": "test",
|
||||
"Metadata": {},
|
||||
"Endpoints": {
|
||||
"docker": {
|
||||
"Host": "unix:///var/run/docker.sock",
|
||||
"SkipTLSVerify": false
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
if not name:
|
||||
name = get_current_context_name()
|
||||
if name == "default":
|
||||
return cls.DEFAULT_CONTEXT
|
||||
return Context.load_context(name)
|
||||
|
||||
@classmethod
|
||||
def contexts(cls):
|
||||
"""Context list.
|
||||
Returns:
|
||||
(Context): List of context objects.
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
names = []
|
||||
for dirname, dirnames, fnames in os.walk(get_meta_dir()):
|
||||
for filename in fnames + dirnames:
|
||||
if filename == METAFILE:
|
||||
try:
|
||||
data = json.load(
|
||||
open(os.path.join(dirname, filename)))
|
||||
names.append(data["Name"])
|
||||
except Exception as e:
|
||||
raise errors.ContextException(
|
||||
f"Failed to load metafile {filename}: {e}",
|
||||
) from e
|
||||
|
||||
contexts = [cls.DEFAULT_CONTEXT]
|
||||
for name in names:
|
||||
contexts.append(Context.load_context(name))
|
||||
return contexts
|
||||
|
||||
@classmethod
|
||||
def get_current_context(cls):
|
||||
"""Get current context.
|
||||
Returns:
|
||||
(Context): current context object.
|
||||
"""
|
||||
return cls.get_context()
|
||||
|
||||
@classmethod
|
||||
def set_current_context(cls, name="default"):
|
||||
ctx = cls.get_context(name)
|
||||
if not ctx:
|
||||
raise errors.ContextNotFound(name)
|
||||
|
||||
err = write_context_name_to_docker_config(name)
|
||||
if err:
|
||||
raise errors.ContextException(
|
||||
f'Failed to set current context: {err}')
|
||||
|
||||
@classmethod
|
||||
def remove_context(cls, name):
|
||||
"""Remove a context. Similar to the ``docker context rm`` command.
|
||||
|
||||
Args:
|
||||
name (str): The name of the context
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.MissingContextParameter`
|
||||
If a context name is not provided.
|
||||
:py:class:`docker.errors.ContextNotFound`
|
||||
If a context with the name does not exist.
|
||||
:py:class:`docker.errors.ContextException`
|
||||
If name is default.
|
||||
|
||||
Example:
|
||||
|
||||
>>> from docker.context import ContextAPI
|
||||
>>> ContextAPI.remove_context(name='test')
|
||||
>>>
|
||||
"""
|
||||
if not name:
|
||||
raise errors.MissingContextParameter("name")
|
||||
if name == "default":
|
||||
raise errors.ContextException(
|
||||
'context "default" cannot be removed')
|
||||
ctx = Context.load_context(name)
|
||||
if not ctx:
|
||||
raise errors.ContextNotFound(name)
|
||||
if name == get_current_context_name():
|
||||
write_context_name_to_docker_config(None)
|
||||
ctx.remove()
|
||||
|
||||
@classmethod
|
||||
def inspect_context(cls, name="default"):
|
||||
"""Remove a context. Similar to the ``docker context inspect`` command.
|
||||
|
||||
Args:
|
||||
name (str): The name of the context
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.MissingContextParameter`
|
||||
If a context name is not provided.
|
||||
:py:class:`docker.errors.ContextNotFound`
|
||||
If a context with the name does not exist.
|
||||
|
||||
Example:
|
||||
|
||||
>>> from docker.context import ContextAPI
|
||||
>>> ContextAPI.remove_context(name='test')
|
||||
>>>
|
||||
"""
|
||||
if not name:
|
||||
raise errors.MissingContextParameter("name")
|
||||
if name == "default":
|
||||
return cls.DEFAULT_CONTEXT()
|
||||
ctx = Context.load_context(name)
|
||||
if not ctx:
|
||||
raise errors.ContextNotFound(name)
|
||||
|
||||
return ctx()
|
||||
@@ -0,0 +1,81 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
|
||||
from docker import utils
|
||||
from docker.constants import DEFAULT_UNIX_SOCKET, IS_WINDOWS_PLATFORM
|
||||
from docker.utils.config import find_config_file
|
||||
|
||||
METAFILE = "meta.json"
|
||||
|
||||
|
||||
def get_current_context_name():
|
||||
name = "default"
|
||||
docker_cfg_path = find_config_file()
|
||||
if docker_cfg_path:
|
||||
try:
|
||||
with open(docker_cfg_path) as f:
|
||||
name = json.load(f).get("currentContext", "default")
|
||||
except Exception:
|
||||
return "default"
|
||||
return name
|
||||
|
||||
|
||||
def write_context_name_to_docker_config(name=None):
|
||||
if name == 'default':
|
||||
name = None
|
||||
docker_cfg_path = find_config_file()
|
||||
config = {}
|
||||
if docker_cfg_path:
|
||||
try:
|
||||
with open(docker_cfg_path) as f:
|
||||
config = json.load(f)
|
||||
except Exception as e:
|
||||
return e
|
||||
current_context = config.get("currentContext", None)
|
||||
if current_context and not name:
|
||||
del config["currentContext"]
|
||||
elif name:
|
||||
config["currentContext"] = name
|
||||
else:
|
||||
return
|
||||
try:
|
||||
with open(docker_cfg_path, "w") as f:
|
||||
json.dump(config, f, indent=4)
|
||||
except Exception as e:
|
||||
return e
|
||||
|
||||
|
||||
def get_context_id(name):
|
||||
return hashlib.sha256(name.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
def get_context_dir():
|
||||
return os.path.join(os.path.dirname(find_config_file() or ""), "contexts")
|
||||
|
||||
|
||||
def get_meta_dir(name=None):
|
||||
meta_dir = os.path.join(get_context_dir(), "meta")
|
||||
if name:
|
||||
return os.path.join(meta_dir, get_context_id(name))
|
||||
return meta_dir
|
||||
|
||||
|
||||
def get_meta_file(name):
|
||||
return os.path.join(get_meta_dir(name), METAFILE)
|
||||
|
||||
|
||||
def get_tls_dir(name=None, endpoint=""):
|
||||
context_dir = get_context_dir()
|
||||
if name:
|
||||
return os.path.join(context_dir, "tls", get_context_id(name), endpoint)
|
||||
return os.path.join(context_dir, "tls")
|
||||
|
||||
|
||||
def get_context_host(path=None, tls=False):
|
||||
host = utils.parse_host(path, IS_WINDOWS_PLATFORM, tls)
|
||||
if host == DEFAULT_UNIX_SOCKET:
|
||||
# remove http+ from default docker socket url
|
||||
if host.startswith("http+"):
|
||||
host = host[5:]
|
||||
return host
|
||||
@@ -0,0 +1,249 @@
|
||||
import json
|
||||
import os
|
||||
from shutil import copyfile, rmtree
|
||||
|
||||
from docker.errors import ContextException
|
||||
from docker.tls import TLSConfig
|
||||
|
||||
from .config import (
|
||||
get_context_host,
|
||||
get_meta_dir,
|
||||
get_meta_file,
|
||||
get_tls_dir,
|
||||
)
|
||||
|
||||
|
||||
class Context:
|
||||
"""A context."""
|
||||
|
||||
def __init__(self, name, orchestrator=None, host=None, endpoints=None,
|
||||
tls=False):
|
||||
if not name:
|
||||
raise Exception("Name not provided")
|
||||
self.name = name
|
||||
self.context_type = None
|
||||
self.orchestrator = orchestrator
|
||||
self.endpoints = {}
|
||||
self.tls_cfg = {}
|
||||
self.meta_path = "IN MEMORY"
|
||||
self.tls_path = "IN MEMORY"
|
||||
|
||||
if not endpoints:
|
||||
# set default docker endpoint if no endpoint is set
|
||||
default_endpoint = "docker" if (
|
||||
not orchestrator or orchestrator == "swarm"
|
||||
) else orchestrator
|
||||
|
||||
self.endpoints = {
|
||||
default_endpoint: {
|
||||
"Host": get_context_host(host, tls),
|
||||
"SkipTLSVerify": not tls
|
||||
}
|
||||
}
|
||||
return
|
||||
|
||||
# check docker endpoints
|
||||
for k, v in endpoints.items():
|
||||
if not isinstance(v, dict):
|
||||
# unknown format
|
||||
raise ContextException(
|
||||
f"Unknown endpoint format for context {name}: {v}",
|
||||
)
|
||||
|
||||
self.endpoints[k] = v
|
||||
if k != "docker":
|
||||
continue
|
||||
|
||||
self.endpoints[k]["Host"] = v.get("Host", get_context_host(
|
||||
host, tls))
|
||||
self.endpoints[k]["SkipTLSVerify"] = bool(v.get(
|
||||
"SkipTLSVerify", not tls))
|
||||
|
||||
def set_endpoint(
|
||||
self, name="docker", host=None, tls_cfg=None,
|
||||
skip_tls_verify=False, def_namespace=None):
|
||||
self.endpoints[name] = {
|
||||
"Host": get_context_host(host, not skip_tls_verify),
|
||||
"SkipTLSVerify": skip_tls_verify
|
||||
}
|
||||
if def_namespace:
|
||||
self.endpoints[name]["DefaultNamespace"] = def_namespace
|
||||
|
||||
if tls_cfg:
|
||||
self.tls_cfg[name] = tls_cfg
|
||||
|
||||
def inspect(self):
|
||||
return self.__call__()
|
||||
|
||||
@classmethod
|
||||
def load_context(cls, name):
|
||||
meta = Context._load_meta(name)
|
||||
if meta:
|
||||
instance = cls(
|
||||
meta["Name"],
|
||||
orchestrator=meta["Metadata"].get("StackOrchestrator", None),
|
||||
endpoints=meta.get("Endpoints", None))
|
||||
instance.context_type = meta["Metadata"].get("Type", None)
|
||||
instance._load_certs()
|
||||
instance.meta_path = get_meta_dir(name)
|
||||
return instance
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _load_meta(cls, name):
|
||||
meta_file = get_meta_file(name)
|
||||
if not os.path.isfile(meta_file):
|
||||
return None
|
||||
|
||||
metadata = {}
|
||||
try:
|
||||
with open(meta_file) as f:
|
||||
metadata = json.load(f)
|
||||
except (OSError, KeyError, ValueError) as e:
|
||||
# unknown format
|
||||
raise Exception(
|
||||
f"Detected corrupted meta file for context {name} : {e}"
|
||||
) from e
|
||||
|
||||
# for docker endpoints, set defaults for
|
||||
# Host and SkipTLSVerify fields
|
||||
for k, v in metadata["Endpoints"].items():
|
||||
if k != "docker":
|
||||
continue
|
||||
metadata["Endpoints"][k]["Host"] = v.get(
|
||||
"Host", get_context_host(None, False))
|
||||
metadata["Endpoints"][k]["SkipTLSVerify"] = bool(
|
||||
v.get("SkipTLSVerify", True))
|
||||
|
||||
return metadata
|
||||
|
||||
def _load_certs(self):
|
||||
certs = {}
|
||||
tls_dir = get_tls_dir(self.name)
|
||||
for endpoint in self.endpoints.keys():
|
||||
if not os.path.isdir(os.path.join(tls_dir, endpoint)):
|
||||
continue
|
||||
ca_cert = None
|
||||
cert = None
|
||||
key = None
|
||||
for filename in os.listdir(os.path.join(tls_dir, endpoint)):
|
||||
if filename.startswith("ca"):
|
||||
ca_cert = os.path.join(tls_dir, endpoint, filename)
|
||||
elif filename.startswith("cert"):
|
||||
cert = os.path.join(tls_dir, endpoint, filename)
|
||||
elif filename.startswith("key"):
|
||||
key = os.path.join(tls_dir, endpoint, filename)
|
||||
if all([ca_cert, cert, key]):
|
||||
verify = None
|
||||
if endpoint == "docker" and not self.endpoints["docker"].get(
|
||||
"SkipTLSVerify", False):
|
||||
verify = True
|
||||
certs[endpoint] = TLSConfig(
|
||||
client_cert=(cert, key), ca_cert=ca_cert, verify=verify)
|
||||
self.tls_cfg = certs
|
||||
self.tls_path = tls_dir
|
||||
|
||||
def save(self):
|
||||
meta_dir = get_meta_dir(self.name)
|
||||
if not os.path.isdir(meta_dir):
|
||||
os.makedirs(meta_dir)
|
||||
with open(get_meta_file(self.name), "w") as f:
|
||||
f.write(json.dumps(self.Metadata))
|
||||
|
||||
tls_dir = get_tls_dir(self.name)
|
||||
for endpoint, tls in self.tls_cfg.items():
|
||||
if not os.path.isdir(os.path.join(tls_dir, endpoint)):
|
||||
os.makedirs(os.path.join(tls_dir, endpoint))
|
||||
|
||||
ca_file = tls.ca_cert
|
||||
if ca_file:
|
||||
copyfile(ca_file, os.path.join(
|
||||
tls_dir, endpoint, os.path.basename(ca_file)))
|
||||
|
||||
if tls.cert:
|
||||
cert_file, key_file = tls.cert
|
||||
copyfile(cert_file, os.path.join(
|
||||
tls_dir, endpoint, os.path.basename(cert_file)))
|
||||
copyfile(key_file, os.path.join(
|
||||
tls_dir, endpoint, os.path.basename(key_file)))
|
||||
|
||||
self.meta_path = get_meta_dir(self.name)
|
||||
self.tls_path = get_tls_dir(self.name)
|
||||
|
||||
def remove(self):
|
||||
if os.path.isdir(self.meta_path):
|
||||
rmtree(self.meta_path)
|
||||
if os.path.isdir(self.tls_path):
|
||||
rmtree(self.tls_path)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<{self.__class__.__name__}: '{self.name}'>"
|
||||
|
||||
def __str__(self):
|
||||
return json.dumps(self.__call__(), indent=2)
|
||||
|
||||
def __call__(self):
|
||||
result = self.Metadata
|
||||
result.update(self.TLSMaterial)
|
||||
result.update(self.Storage)
|
||||
return result
|
||||
|
||||
def is_docker_host(self):
|
||||
return self.context_type is None
|
||||
|
||||
@property
|
||||
def Name(self):
|
||||
return self.name
|
||||
|
||||
@property
|
||||
def Host(self):
|
||||
if not self.orchestrator or self.orchestrator == "swarm":
|
||||
endpoint = self.endpoints.get("docker", None)
|
||||
if endpoint:
|
||||
return endpoint.get("Host", None)
|
||||
return None
|
||||
|
||||
return self.endpoints[self.orchestrator].get("Host", None)
|
||||
|
||||
@property
|
||||
def Orchestrator(self):
|
||||
return self.orchestrator
|
||||
|
||||
@property
|
||||
def Metadata(self):
|
||||
meta = {}
|
||||
if self.orchestrator:
|
||||
meta = {"StackOrchestrator": self.orchestrator}
|
||||
return {
|
||||
"Name": self.name,
|
||||
"Metadata": meta,
|
||||
"Endpoints": self.endpoints
|
||||
}
|
||||
|
||||
@property
|
||||
def TLSConfig(self):
|
||||
key = self.orchestrator
|
||||
if not key or key == "swarm":
|
||||
key = "docker"
|
||||
if key in self.tls_cfg.keys():
|
||||
return self.tls_cfg[key]
|
||||
return None
|
||||
|
||||
@property
|
||||
def TLSMaterial(self):
|
||||
certs = {}
|
||||
for endpoint, tls in self.tls_cfg.items():
|
||||
cert, key = tls.cert
|
||||
certs[endpoint] = list(
|
||||
map(os.path.basename, [tls.ca_cert, cert, key]))
|
||||
return {
|
||||
"TLSMaterial": certs
|
||||
}
|
||||
|
||||
@property
|
||||
def Storage(self):
|
||||
return {
|
||||
"Storage": {
|
||||
"MetadataPath": self.meta_path,
|
||||
"TLSPath": self.tls_path
|
||||
}}
|
||||
@@ -0,0 +1,8 @@
|
||||
from .constants import (
|
||||
DEFAULT_LINUX_STORE,
|
||||
DEFAULT_OSX_STORE,
|
||||
DEFAULT_WIN32_STORE,
|
||||
PROGRAM_PREFIX,
|
||||
)
|
||||
from .errors import CredentialsNotFound, StoreError
|
||||
from .store import Store
|
||||
@@ -0,0 +1,4 @@
|
||||
PROGRAM_PREFIX = 'docker-credential-'
|
||||
DEFAULT_LINUX_STORE = 'secretservice'
|
||||
DEFAULT_OSX_STORE = 'osxkeychain'
|
||||
DEFAULT_WIN32_STORE = 'wincred'
|
||||
@@ -0,0 +1,17 @@
|
||||
class StoreError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class CredentialsNotFound(StoreError):
|
||||
pass
|
||||
|
||||
|
||||
class InitializationError(StoreError):
|
||||
pass
|
||||
|
||||
|
||||
def process_store_error(cpe, program):
|
||||
message = cpe.output.decode('utf-8')
|
||||
if 'credentials not found in native keychain' in message:
|
||||
return CredentialsNotFound(f'No matching credentials in {program}')
|
||||
return StoreError(f'Credentials store {program} exited with "{message}".')
|
||||
@@ -0,0 +1,93 @@
|
||||
import errno
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import warnings
|
||||
|
||||
from . import constants, errors
|
||||
from .utils import create_environment_dict
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, program, environment=None):
|
||||
""" Create a store object that acts as an interface to
|
||||
perform the basic operations for storing, retrieving
|
||||
and erasing credentials using `program`.
|
||||
"""
|
||||
self.program = constants.PROGRAM_PREFIX + program
|
||||
self.exe = shutil.which(self.program)
|
||||
self.environment = environment
|
||||
if self.exe is None:
|
||||
warnings.warn(
|
||||
f'{self.program} not installed or not available in PATH',
|
||||
stacklevel=1,
|
||||
)
|
||||
|
||||
def get(self, server):
|
||||
""" Retrieve credentials for `server`. If no credentials are found,
|
||||
a `StoreError` will be raised.
|
||||
"""
|
||||
if not isinstance(server, bytes):
|
||||
server = server.encode('utf-8')
|
||||
data = self._execute('get', server)
|
||||
result = json.loads(data.decode('utf-8'))
|
||||
|
||||
# docker-credential-pass will return an object for inexistent servers
|
||||
# whereas other helpers will exit with returncode != 0. For
|
||||
# consistency, if no significant data is returned,
|
||||
# raise CredentialsNotFound
|
||||
if result['Username'] == '' and result['Secret'] == '':
|
||||
raise errors.CredentialsNotFound(
|
||||
f'No matching credentials in {self.program}'
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def store(self, server, username, secret):
|
||||
""" Store credentials for `server`. Raises a `StoreError` if an error
|
||||
occurs.
|
||||
"""
|
||||
data_input = json.dumps({
|
||||
'ServerURL': server,
|
||||
'Username': username,
|
||||
'Secret': secret
|
||||
}).encode('utf-8')
|
||||
return self._execute('store', data_input)
|
||||
|
||||
def erase(self, server):
|
||||
""" Erase credentials for `server`. Raises a `StoreError` if an error
|
||||
occurs.
|
||||
"""
|
||||
if not isinstance(server, bytes):
|
||||
server = server.encode('utf-8')
|
||||
self._execute('erase', server)
|
||||
|
||||
def list(self):
|
||||
""" List stored credentials. Requires v0.4.0+ of the helper.
|
||||
"""
|
||||
data = self._execute('list', None)
|
||||
return json.loads(data.decode('utf-8'))
|
||||
|
||||
def _execute(self, subcmd, data_input):
|
||||
if self.exe is None:
|
||||
raise errors.StoreError(
|
||||
f'{self.program} not installed or not available in PATH'
|
||||
)
|
||||
output = None
|
||||
env = create_environment_dict(self.environment)
|
||||
try:
|
||||
output = subprocess.check_output(
|
||||
[self.exe, subcmd], input=data_input, env=env,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise errors.process_store_error(e, self.program) from e
|
||||
except OSError as e:
|
||||
if e.errno == errno.ENOENT:
|
||||
raise errors.StoreError(
|
||||
f'{self.program} not installed or not available in PATH'
|
||||
) from e
|
||||
else:
|
||||
raise errors.StoreError(
|
||||
f'Unexpected OS error "{e.strerror}", errno={e.errno}'
|
||||
) from e
|
||||
return output
|
||||
@@ -0,0 +1,10 @@
|
||||
import os
|
||||
|
||||
|
||||
def create_environment_dict(overrides):
|
||||
"""
|
||||
Create and return a copy of os.environ with the specified overrides
|
||||
"""
|
||||
result = os.environ.copy()
|
||||
result.update(overrides or {})
|
||||
return result
|
||||
@@ -0,0 +1,209 @@
|
||||
import requests
|
||||
|
||||
_image_not_found_explanation_fragments = frozenset(
|
||||
fragment.lower() for fragment in [
|
||||
'no such image',
|
||||
'not found: does not exist or no pull access',
|
||||
'repository does not exist',
|
||||
'was found but does not match the specified platform',
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class DockerException(Exception):
|
||||
"""
|
||||
A base class from which all other exceptions inherit.
|
||||
|
||||
If you want to catch all errors that the Docker SDK might raise,
|
||||
catch this base exception.
|
||||
"""
|
||||
|
||||
|
||||
def create_api_error_from_http_exception(e):
|
||||
"""
|
||||
Create a suitable APIError from requests.exceptions.HTTPError.
|
||||
"""
|
||||
response = e.response
|
||||
try:
|
||||
explanation = response.json()['message']
|
||||
except ValueError:
|
||||
explanation = (response.text or '').strip()
|
||||
cls = APIError
|
||||
if response.status_code == 404:
|
||||
explanation_msg = (explanation or '').lower()
|
||||
if any(fragment in explanation_msg
|
||||
for fragment in _image_not_found_explanation_fragments):
|
||||
cls = ImageNotFound
|
||||
else:
|
||||
cls = NotFound
|
||||
raise cls(e, response=response, explanation=explanation) from e
|
||||
|
||||
|
||||
class APIError(requests.exceptions.HTTPError, DockerException):
|
||||
"""
|
||||
An HTTP error from the API.
|
||||
"""
|
||||
def __init__(self, message, response=None, explanation=None):
|
||||
# requests 1.2 supports response as a keyword argument, but
|
||||
# requests 1.1 doesn't
|
||||
super().__init__(message)
|
||||
self.response = response
|
||||
self.explanation = explanation
|
||||
|
||||
def __str__(self):
|
||||
message = super().__str__()
|
||||
|
||||
if self.is_client_error():
|
||||
message = (
|
||||
f'{self.response.status_code} Client Error for '
|
||||
f'{self.response.url}: {self.response.reason}'
|
||||
)
|
||||
|
||||
elif self.is_server_error():
|
||||
message = (
|
||||
f'{self.response.status_code} Server Error for '
|
||||
f'{self.response.url}: {self.response.reason}'
|
||||
)
|
||||
|
||||
if self.explanation:
|
||||
message = f'{message} ("{self.explanation}")'
|
||||
|
||||
return message
|
||||
|
||||
@property
|
||||
def status_code(self):
|
||||
if self.response is not None:
|
||||
return self.response.status_code
|
||||
|
||||
def is_error(self):
|
||||
return self.is_client_error() or self.is_server_error()
|
||||
|
||||
def is_client_error(self):
|
||||
if self.status_code is None:
|
||||
return False
|
||||
return 400 <= self.status_code < 500
|
||||
|
||||
def is_server_error(self):
|
||||
if self.status_code is None:
|
||||
return False
|
||||
return 500 <= self.status_code < 600
|
||||
|
||||
|
||||
class NotFound(APIError):
|
||||
pass
|
||||
|
||||
|
||||
class ImageNotFound(NotFound):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidVersion(DockerException):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidRepository(DockerException):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidConfigFile(DockerException):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidArgument(DockerException):
|
||||
pass
|
||||
|
||||
|
||||
class DeprecatedMethod(DockerException):
|
||||
pass
|
||||
|
||||
|
||||
class TLSParameterError(DockerException):
|
||||
def __init__(self, msg):
|
||||
self.msg = msg
|
||||
|
||||
def __str__(self):
|
||||
return self.msg + (". TLS configurations should map the Docker CLI "
|
||||
"client configurations. See "
|
||||
"https://docs.docker.com/engine/articles/https/ "
|
||||
"for API details.")
|
||||
|
||||
|
||||
class NullResource(DockerException, ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ContainerError(DockerException):
|
||||
"""
|
||||
Represents a container that has exited with a non-zero exit code.
|
||||
"""
|
||||
def __init__(self, container, exit_status, command, image, stderr):
|
||||
self.container = container
|
||||
self.exit_status = exit_status
|
||||
self.command = command
|
||||
self.image = image
|
||||
self.stderr = stderr
|
||||
|
||||
err = f": {stderr}" if stderr is not None else ""
|
||||
super().__init__(
|
||||
f"Command '{command}' in image '{image}' "
|
||||
f"returned non-zero exit status {exit_status}{err}"
|
||||
)
|
||||
|
||||
|
||||
class StreamParseError(RuntimeError):
|
||||
def __init__(self, reason):
|
||||
self.msg = reason
|
||||
|
||||
|
||||
class BuildError(DockerException):
|
||||
def __init__(self, reason, build_log):
|
||||
super().__init__(reason)
|
||||
self.msg = reason
|
||||
self.build_log = build_log
|
||||
|
||||
|
||||
class ImageLoadError(DockerException):
|
||||
pass
|
||||
|
||||
|
||||
def create_unexpected_kwargs_error(name, kwargs):
|
||||
quoted_kwargs = [f"'{k}'" for k in sorted(kwargs)]
|
||||
text = [f"{name}() "]
|
||||
if len(quoted_kwargs) == 1:
|
||||
text.append("got an unexpected keyword argument ")
|
||||
else:
|
||||
text.append("got unexpected keyword arguments ")
|
||||
text.append(', '.join(quoted_kwargs))
|
||||
return TypeError(''.join(text))
|
||||
|
||||
|
||||
class MissingContextParameter(DockerException):
|
||||
def __init__(self, param):
|
||||
self.param = param
|
||||
|
||||
def __str__(self):
|
||||
return (f"missing parameter: {self.param}")
|
||||
|
||||
|
||||
class ContextAlreadyExists(DockerException):
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __str__(self):
|
||||
return (f"context {self.name} already exists")
|
||||
|
||||
|
||||
class ContextException(DockerException):
|
||||
def __init__(self, msg):
|
||||
self.msg = msg
|
||||
|
||||
def __str__(self):
|
||||
return (self.msg)
|
||||
|
||||
|
||||
class ContextNotFound(DockerException):
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __str__(self):
|
||||
return (f"context '{self.name}' not found")
|
||||
@@ -0,0 +1,70 @@
|
||||
from ..api import APIClient
|
||||
from .resource import Collection, Model
|
||||
|
||||
|
||||
class Config(Model):
|
||||
"""A config."""
|
||||
id_attribute = 'ID'
|
||||
|
||||
def __repr__(self):
|
||||
return f"<{self.__class__.__name__}: '{self.name}'>"
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.attrs['Spec']['Name']
|
||||
|
||||
def remove(self):
|
||||
"""
|
||||
Remove this config.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If config failed to remove.
|
||||
"""
|
||||
return self.client.api.remove_config(self.id)
|
||||
|
||||
|
||||
class ConfigCollection(Collection):
|
||||
"""Configs on the Docker server."""
|
||||
model = Config
|
||||
|
||||
def create(self, **kwargs):
|
||||
obj = self.client.api.create_config(**kwargs)
|
||||
obj.setdefault("Spec", {})["Name"] = kwargs.get("name")
|
||||
return self.prepare_model(obj)
|
||||
create.__doc__ = APIClient.create_config.__doc__
|
||||
|
||||
def get(self, config_id):
|
||||
"""
|
||||
Get a config.
|
||||
|
||||
Args:
|
||||
config_id (str): Config ID.
|
||||
|
||||
Returns:
|
||||
(:py:class:`Config`): The config.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.NotFound`
|
||||
If the config does not exist.
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.prepare_model(self.client.api.inspect_config(config_id))
|
||||
|
||||
def list(self, **kwargs):
|
||||
"""
|
||||
List configs. Similar to the ``docker config ls`` command.
|
||||
|
||||
Args:
|
||||
filters (dict): Server-side list filtering options.
|
||||
|
||||
Returns:
|
||||
(list of :py:class:`Config`): The configs.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
resp = self.client.api.configs(**kwargs)
|
||||
return [self.prepare_model(obj) for obj in resp]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,505 @@
|
||||
import itertools
|
||||
import re
|
||||
import warnings
|
||||
|
||||
from ..api import APIClient
|
||||
from ..constants import DEFAULT_DATA_CHUNK_SIZE
|
||||
from ..errors import BuildError, ImageLoadError, InvalidArgument
|
||||
from ..utils import parse_repository_tag
|
||||
from ..utils.json_stream import json_stream
|
||||
from .resource import Collection, Model
|
||||
|
||||
|
||||
class Image(Model):
|
||||
"""
|
||||
An image on the server.
|
||||
"""
|
||||
def __repr__(self):
|
||||
tag_str = "', '".join(self.tags)
|
||||
return f"<{self.__class__.__name__}: '{tag_str}'>"
|
||||
|
||||
@property
|
||||
def labels(self):
|
||||
"""
|
||||
The labels of an image as dictionary.
|
||||
"""
|
||||
result = self.attrs['Config'].get('Labels')
|
||||
return result or {}
|
||||
|
||||
@property
|
||||
def short_id(self):
|
||||
"""
|
||||
The ID of the image truncated to 12 characters, plus the ``sha256:``
|
||||
prefix.
|
||||
"""
|
||||
if self.id.startswith('sha256:'):
|
||||
return self.id[:19]
|
||||
return self.id[:12]
|
||||
|
||||
@property
|
||||
def tags(self):
|
||||
"""
|
||||
The image's tags.
|
||||
"""
|
||||
tags = self.attrs.get('RepoTags')
|
||||
if tags is None:
|
||||
tags = []
|
||||
return [tag for tag in tags if tag != '<none>:<none>']
|
||||
|
||||
def history(self):
|
||||
"""
|
||||
Show the history of an image.
|
||||
|
||||
Returns:
|
||||
(list): The history of the image.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.client.api.history(self.id)
|
||||
|
||||
def remove(self, force=False, noprune=False):
|
||||
"""
|
||||
Remove this image.
|
||||
|
||||
Args:
|
||||
force (bool): Force removal of the image
|
||||
noprune (bool): Do not delete untagged parents
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.client.api.remove_image(
|
||||
self.id,
|
||||
force=force,
|
||||
noprune=noprune,
|
||||
)
|
||||
|
||||
def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE, named=False):
|
||||
"""
|
||||
Get a tarball of an image. Similar to the ``docker save`` command.
|
||||
|
||||
Args:
|
||||
chunk_size (int): The generator will return up to that much data
|
||||
per iteration, but may return less. If ``None``, data will be
|
||||
streamed as it is received. Default: 2 MB
|
||||
named (str or bool): If ``False`` (default), the tarball will not
|
||||
retain repository and tag information for this image. If set
|
||||
to ``True``, the first tag in the :py:attr:`~tags` list will
|
||||
be used to identify the image. Alternatively, any element of
|
||||
the :py:attr:`~tags` list can be used as an argument to use
|
||||
that specific tag as the saved identifier.
|
||||
|
||||
Returns:
|
||||
(generator): A stream of raw archive data.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> image = cli.images.get("busybox:latest")
|
||||
>>> f = open('/tmp/busybox-latest.tar', 'wb')
|
||||
>>> for chunk in image.save():
|
||||
>>> f.write(chunk)
|
||||
>>> f.close()
|
||||
"""
|
||||
img = self.id
|
||||
if named:
|
||||
img = self.tags[0] if self.tags else img
|
||||
if isinstance(named, str):
|
||||
if named not in self.tags:
|
||||
raise InvalidArgument(
|
||||
f"{named} is not a valid tag for this image"
|
||||
)
|
||||
img = named
|
||||
|
||||
return self.client.api.get_image(img, chunk_size)
|
||||
|
||||
def tag(self, repository, tag=None, **kwargs):
|
||||
"""
|
||||
Tag this image into a repository. Similar to the ``docker tag``
|
||||
command.
|
||||
|
||||
Args:
|
||||
repository (str): The repository to set for the tag
|
||||
tag (str): The tag name
|
||||
force (bool): Force
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Returns:
|
||||
(bool): ``True`` if successful
|
||||
"""
|
||||
return self.client.api.tag(self.id, repository, tag=tag, **kwargs)
|
||||
|
||||
|
||||
class RegistryData(Model):
|
||||
"""
|
||||
Image metadata stored on the registry, including available platforms.
|
||||
"""
|
||||
def __init__(self, image_name, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.image_name = image_name
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
"""
|
||||
The ID of the object.
|
||||
"""
|
||||
return self.attrs['Descriptor']['digest']
|
||||
|
||||
@property
|
||||
def short_id(self):
|
||||
"""
|
||||
The ID of the image truncated to 12 characters, plus the ``sha256:``
|
||||
prefix.
|
||||
"""
|
||||
return self.id[:19]
|
||||
|
||||
def pull(self, platform=None):
|
||||
"""
|
||||
Pull the image digest.
|
||||
|
||||
Args:
|
||||
platform (str): The platform to pull the image for.
|
||||
Default: ``None``
|
||||
|
||||
Returns:
|
||||
(:py:class:`Image`): A reference to the pulled image.
|
||||
"""
|
||||
repository, _ = parse_repository_tag(self.image_name)
|
||||
return self.collection.pull(repository, tag=self.id, platform=platform)
|
||||
|
||||
def has_platform(self, platform):
|
||||
"""
|
||||
Check whether the given platform identifier is available for this
|
||||
digest.
|
||||
|
||||
Args:
|
||||
platform (str or dict): A string using the ``os[/arch[/variant]]``
|
||||
format, or a platform dictionary.
|
||||
|
||||
Returns:
|
||||
(bool): ``True`` if the platform is recognized as available,
|
||||
``False`` otherwise.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.InvalidArgument`
|
||||
If the platform argument is not a valid descriptor.
|
||||
"""
|
||||
if platform and not isinstance(platform, dict):
|
||||
parts = platform.split('/')
|
||||
if len(parts) > 3 or len(parts) < 1:
|
||||
raise InvalidArgument(
|
||||
f'"{platform}" is not a valid platform descriptor'
|
||||
)
|
||||
platform = {'os': parts[0]}
|
||||
if len(parts) > 2:
|
||||
platform['variant'] = parts[2]
|
||||
if len(parts) > 1:
|
||||
platform['architecture'] = parts[1]
|
||||
return normalize_platform(
|
||||
platform, self.client.version()
|
||||
) in self.attrs['Platforms']
|
||||
|
||||
def reload(self):
|
||||
self.attrs = self.client.api.inspect_distribution(self.image_name)
|
||||
|
||||
reload.__doc__ = Model.reload.__doc__
|
||||
|
||||
|
||||
class ImageCollection(Collection):
|
||||
model = Image
|
||||
|
||||
def build(self, **kwargs):
|
||||
"""
|
||||
Build an image and return it. Similar to the ``docker build``
|
||||
command. Either ``path`` or ``fileobj`` must be set.
|
||||
|
||||
If you already have a tar file for the Docker build context (including
|
||||
a Dockerfile), pass a readable file-like object to ``fileobj``
|
||||
and also pass ``custom_context=True``. If the stream is also
|
||||
compressed, set ``encoding`` to the correct value (e.g ``gzip``).
|
||||
|
||||
If you want to get the raw output of the build, use the
|
||||
:py:meth:`~docker.api.build.BuildApiMixin.build` method in the
|
||||
low-level API.
|
||||
|
||||
Args:
|
||||
path (str): Path to the directory containing the Dockerfile
|
||||
fileobj: A file object to use as the Dockerfile. (Or a file-like
|
||||
object)
|
||||
tag (str): A tag to add to the final image
|
||||
quiet (bool): Whether to return the status
|
||||
nocache (bool): Don't use the cache when set to ``True``
|
||||
rm (bool): Remove intermediate containers. The ``docker build``
|
||||
command now defaults to ``--rm=true``, but we have kept the old
|
||||
default of `False` to preserve backward compatibility
|
||||
timeout (int): HTTP timeout
|
||||
custom_context (bool): Optional if using ``fileobj``
|
||||
encoding (str): The encoding for a stream. Set to ``gzip`` for
|
||||
compressing
|
||||
pull (bool): Downloads any updates to the FROM image in Dockerfiles
|
||||
forcerm (bool): Always remove intermediate containers, even after
|
||||
unsuccessful builds
|
||||
dockerfile (str): path within the build context to the Dockerfile
|
||||
buildargs (dict): A dictionary of build arguments
|
||||
container_limits (dict): A dictionary of limits applied to each
|
||||
container created by the build process. Valid keys:
|
||||
|
||||
- memory (int): set memory limit for build
|
||||
- memswap (int): Total memory (memory + swap), -1 to disable
|
||||
swap
|
||||
- cpushares (int): CPU shares (relative weight)
|
||||
- cpusetcpus (str): CPUs in which to allow execution, e.g.,
|
||||
``"0-3"``, ``"0,1"``
|
||||
shmsize (int): Size of `/dev/shm` in bytes. The size must be
|
||||
greater than 0. If omitted the system uses 64MB
|
||||
labels (dict): A dictionary of labels to set on the image
|
||||
cache_from (list): A list of images used for build cache
|
||||
resolution
|
||||
target (str): Name of the build-stage to build in a multi-stage
|
||||
Dockerfile
|
||||
network_mode (str): networking mode for the run commands during
|
||||
build
|
||||
squash (bool): Squash the resulting images layers into a
|
||||
single layer.
|
||||
extra_hosts (dict): Extra hosts to add to /etc/hosts in building
|
||||
containers, as a mapping of hostname to IP address.
|
||||
platform (str): Platform in the format ``os[/arch[/variant]]``.
|
||||
isolation (str): Isolation technology used during build.
|
||||
Default: `None`.
|
||||
use_config_proxy (bool): If ``True``, and if the docker client
|
||||
configuration file (``~/.docker/config.json`` by default)
|
||||
contains a proxy configuration, the corresponding environment
|
||||
variables will be set in the container being built.
|
||||
|
||||
Returns:
|
||||
(tuple): The first item is the :py:class:`Image` object for the
|
||||
image that was built. The second item is a generator of the
|
||||
build logs as JSON-decoded objects.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.BuildError`
|
||||
If there is an error during the build.
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns any other error.
|
||||
``TypeError``
|
||||
If neither ``path`` nor ``fileobj`` is specified.
|
||||
"""
|
||||
resp = self.client.api.build(**kwargs)
|
||||
if isinstance(resp, str):
|
||||
return self.get(resp)
|
||||
last_event = None
|
||||
image_id = None
|
||||
result_stream, internal_stream = itertools.tee(json_stream(resp))
|
||||
for chunk in internal_stream:
|
||||
if 'error' in chunk:
|
||||
raise BuildError(chunk['error'], result_stream)
|
||||
if 'stream' in chunk:
|
||||
match = re.search(
|
||||
r'(^Successfully built |sha256:)([0-9a-f]+)$',
|
||||
chunk['stream']
|
||||
)
|
||||
if match:
|
||||
image_id = match.group(2)
|
||||
last_event = chunk
|
||||
if image_id:
|
||||
return (self.get(image_id), result_stream)
|
||||
raise BuildError(last_event or 'Unknown', result_stream)
|
||||
|
||||
def get(self, name):
|
||||
"""
|
||||
Gets an image.
|
||||
|
||||
Args:
|
||||
name (str): The name of the image.
|
||||
|
||||
Returns:
|
||||
(:py:class:`Image`): The image.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.ImageNotFound`
|
||||
If the image does not exist.
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.prepare_model(self.client.api.inspect_image(name))
|
||||
|
||||
def get_registry_data(self, name, auth_config=None):
|
||||
"""
|
||||
Gets the registry data for an image.
|
||||
|
||||
Args:
|
||||
name (str): The name of the image.
|
||||
auth_config (dict): Override the credentials that are found in the
|
||||
config for this request. ``auth_config`` should contain the
|
||||
``username`` and ``password`` keys to be valid.
|
||||
|
||||
Returns:
|
||||
(:py:class:`RegistryData`): The data object.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return RegistryData(
|
||||
image_name=name,
|
||||
attrs=self.client.api.inspect_distribution(name, auth_config),
|
||||
client=self.client,
|
||||
collection=self,
|
||||
)
|
||||
|
||||
def list(self, name=None, all=False, filters=None):
|
||||
"""
|
||||
List images on the server.
|
||||
|
||||
Args:
|
||||
name (str): Only show images belonging to the repository ``name``
|
||||
all (bool): Show intermediate image layers. By default, these are
|
||||
filtered out.
|
||||
filters (dict): Filters to be processed on the image list.
|
||||
Available filters:
|
||||
- ``dangling`` (bool)
|
||||
- `label` (str|list): format either ``"key"``, ``"key=value"``
|
||||
or a list of such.
|
||||
|
||||
Returns:
|
||||
(list of :py:class:`Image`): The images.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
resp = self.client.api.images(name=name, all=all, filters=filters)
|
||||
return [self.get(r["Id"]) for r in resp]
|
||||
|
||||
def load(self, data):
|
||||
"""
|
||||
Load an image that was previously saved using
|
||||
:py:meth:`~docker.models.images.Image.save` (or ``docker save``).
|
||||
Similar to ``docker load``.
|
||||
|
||||
Args:
|
||||
data (binary): Image data to be loaded.
|
||||
|
||||
Returns:
|
||||
(list of :py:class:`Image`): The images.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
resp = self.client.api.load_image(data)
|
||||
images = []
|
||||
for chunk in resp:
|
||||
if 'stream' in chunk:
|
||||
match = re.search(
|
||||
r'(^Loaded image ID: |^Loaded image: )(.+)$',
|
||||
chunk['stream']
|
||||
)
|
||||
if match:
|
||||
image_id = match.group(2)
|
||||
images.append(image_id)
|
||||
if 'error' in chunk:
|
||||
raise ImageLoadError(chunk['error'])
|
||||
|
||||
return [self.get(i) for i in images]
|
||||
|
||||
def pull(self, repository, tag=None, all_tags=False, **kwargs):
|
||||
"""
|
||||
Pull an image of the given name and return it. Similar to the
|
||||
``docker pull`` command.
|
||||
If ``tag`` is ``None`` or empty, it is set to ``latest``.
|
||||
If ``all_tags`` is set, the ``tag`` parameter is ignored and all image
|
||||
tags will be pulled.
|
||||
|
||||
If you want to get the raw pull output, use the
|
||||
:py:meth:`~docker.api.image.ImageApiMixin.pull` method in the
|
||||
low-level API.
|
||||
|
||||
Args:
|
||||
repository (str): The repository to pull
|
||||
tag (str): The tag to pull
|
||||
auth_config (dict): Override the credentials that are found in the
|
||||
config for this request. ``auth_config`` should contain the
|
||||
``username`` and ``password`` keys to be valid.
|
||||
platform (str): Platform in the format ``os[/arch[/variant]]``
|
||||
all_tags (bool): Pull all image tags
|
||||
|
||||
Returns:
|
||||
(:py:class:`Image` or list): The image that has been pulled.
|
||||
If ``all_tags`` is True, the method will return a list
|
||||
of :py:class:`Image` objects belonging to this repository.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> # Pull the image tagged `latest` in the busybox repo
|
||||
>>> image = client.images.pull('busybox')
|
||||
|
||||
>>> # Pull all tags in the busybox repo
|
||||
>>> images = client.images.pull('busybox', all_tags=True)
|
||||
"""
|
||||
repository, image_tag = parse_repository_tag(repository)
|
||||
tag = tag or image_tag or 'latest'
|
||||
|
||||
if 'stream' in kwargs:
|
||||
warnings.warn(
|
||||
'`stream` is not a valid parameter for this method'
|
||||
' and will be overridden',
|
||||
stacklevel=1,
|
||||
)
|
||||
del kwargs['stream']
|
||||
|
||||
pull_log = self.client.api.pull(
|
||||
repository, tag=tag, stream=True, all_tags=all_tags, **kwargs
|
||||
)
|
||||
for _ in pull_log:
|
||||
# We don't do anything with the logs, but we need
|
||||
# to keep the connection alive and wait for the image
|
||||
# to be pulled.
|
||||
pass
|
||||
if not all_tags:
|
||||
sep = '@' if tag.startswith('sha256:') else ':'
|
||||
return self.get(f'{repository}{sep}{tag}')
|
||||
return self.list(repository)
|
||||
|
||||
def push(self, repository, tag=None, **kwargs):
|
||||
return self.client.api.push(repository, tag=tag, **kwargs)
|
||||
push.__doc__ = APIClient.push.__doc__
|
||||
|
||||
def remove(self, *args, **kwargs):
|
||||
self.client.api.remove_image(*args, **kwargs)
|
||||
remove.__doc__ = APIClient.remove_image.__doc__
|
||||
|
||||
def search(self, *args, **kwargs):
|
||||
return self.client.api.search(*args, **kwargs)
|
||||
search.__doc__ = APIClient.search.__doc__
|
||||
|
||||
def prune(self, filters=None):
|
||||
return self.client.api.prune_images(filters=filters)
|
||||
prune.__doc__ = APIClient.prune_images.__doc__
|
||||
|
||||
def prune_builds(self, *args, **kwargs):
|
||||
return self.client.api.prune_builds(*args, **kwargs)
|
||||
prune_builds.__doc__ = APIClient.prune_builds.__doc__
|
||||
|
||||
|
||||
def normalize_platform(platform, engine_info):
|
||||
if platform is None:
|
||||
platform = {}
|
||||
if 'os' not in platform:
|
||||
platform['os'] = engine_info['Os']
|
||||
if 'architecture' not in platform:
|
||||
platform['architecture'] = engine_info['Arch']
|
||||
return platform
|
||||
@@ -0,0 +1,218 @@
|
||||
from ..api import APIClient
|
||||
from ..utils import version_gte
|
||||
from .containers import Container
|
||||
from .resource import Collection, Model
|
||||
|
||||
|
||||
class Network(Model):
|
||||
"""
|
||||
A Docker network.
|
||||
"""
|
||||
@property
|
||||
def name(self):
|
||||
"""
|
||||
The name of the network.
|
||||
"""
|
||||
return self.attrs.get('Name')
|
||||
|
||||
@property
|
||||
def containers(self):
|
||||
"""
|
||||
The containers that are connected to the network, as a list of
|
||||
:py:class:`~docker.models.containers.Container` objects.
|
||||
"""
|
||||
return [
|
||||
self.client.containers.get(cid) for cid in
|
||||
(self.attrs.get('Containers') or {}).keys()
|
||||
]
|
||||
|
||||
def connect(self, container, *args, **kwargs):
|
||||
"""
|
||||
Connect a container to this network.
|
||||
|
||||
Args:
|
||||
container (str): Container to connect to this network, as either
|
||||
an ID, name, or :py:class:`~docker.models.containers.Container`
|
||||
object.
|
||||
aliases (:py:class:`list`): A list of aliases for this endpoint.
|
||||
Names in that list can be used within the network to reach the
|
||||
container. Defaults to ``None``.
|
||||
links (:py:class:`list`): A list of links for this endpoint.
|
||||
Containers declared in this list will be linkedto this
|
||||
container. Defaults to ``None``.
|
||||
ipv4_address (str): The IP address of this container on the
|
||||
network, using the IPv4 protocol. Defaults to ``None``.
|
||||
ipv6_address (str): The IP address of this container on the
|
||||
network, using the IPv6 protocol. Defaults to ``None``.
|
||||
link_local_ips (:py:class:`list`): A list of link-local (IPv4/IPv6)
|
||||
addresses.
|
||||
driver_opt (dict): A dictionary of options to provide to the
|
||||
network driver. Defaults to ``None``.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
if isinstance(container, Container):
|
||||
container = container.id
|
||||
return self.client.api.connect_container_to_network(
|
||||
container, self.id, *args, **kwargs
|
||||
)
|
||||
|
||||
def disconnect(self, container, *args, **kwargs):
|
||||
"""
|
||||
Disconnect a container from this network.
|
||||
|
||||
Args:
|
||||
container (str): Container to disconnect from this network, as
|
||||
either an ID, name, or
|
||||
:py:class:`~docker.models.containers.Container` object.
|
||||
force (bool): Force the container to disconnect from a network.
|
||||
Default: ``False``
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
if isinstance(container, Container):
|
||||
container = container.id
|
||||
return self.client.api.disconnect_container_from_network(
|
||||
container, self.id, *args, **kwargs
|
||||
)
|
||||
|
||||
def remove(self):
|
||||
"""
|
||||
Remove this network.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.client.api.remove_network(self.id)
|
||||
|
||||
|
||||
class NetworkCollection(Collection):
|
||||
"""
|
||||
Networks on the Docker server.
|
||||
"""
|
||||
model = Network
|
||||
|
||||
def create(self, name, *args, **kwargs):
|
||||
"""
|
||||
Create a network. Similar to the ``docker network create``.
|
||||
|
||||
Args:
|
||||
name (str): Name of the network
|
||||
driver (str): Name of the driver used to create the network
|
||||
options (dict): Driver options as a key-value dictionary
|
||||
ipam (IPAMConfig): Optional custom IP scheme for the network.
|
||||
check_duplicate (bool): Request daemon to check for networks with
|
||||
same name. Default: ``None``.
|
||||
internal (bool): Restrict external access to the network. Default
|
||||
``False``.
|
||||
labels (dict): Map of labels to set on the network. Default
|
||||
``None``.
|
||||
enable_ipv6 (bool): Enable IPv6 on the network. Default ``False``.
|
||||
attachable (bool): If enabled, and the network is in the global
|
||||
scope, non-service containers on worker nodes will be able to
|
||||
connect to the network.
|
||||
scope (str): Specify the network's scope (``local``, ``global`` or
|
||||
``swarm``)
|
||||
ingress (bool): If set, create an ingress network which provides
|
||||
the routing-mesh in swarm mode.
|
||||
|
||||
Returns:
|
||||
(:py:class:`Network`): The network that was created.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
A network using the bridge driver:
|
||||
|
||||
>>> client.networks.create("network1", driver="bridge")
|
||||
|
||||
You can also create more advanced networks with custom IPAM
|
||||
configurations. For example, setting the subnet to
|
||||
``192.168.52.0/24`` and gateway address to ``192.168.52.254``.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
>>> ipam_pool = docker.types.IPAMPool(
|
||||
subnet='192.168.52.0/24',
|
||||
gateway='192.168.52.254'
|
||||
)
|
||||
>>> ipam_config = docker.types.IPAMConfig(
|
||||
pool_configs=[ipam_pool]
|
||||
)
|
||||
>>> client.networks.create(
|
||||
"network1",
|
||||
driver="bridge",
|
||||
ipam=ipam_config
|
||||
)
|
||||
|
||||
"""
|
||||
resp = self.client.api.create_network(name, *args, **kwargs)
|
||||
return self.get(resp['Id'])
|
||||
|
||||
def get(self, network_id, *args, **kwargs):
|
||||
"""
|
||||
Get a network by its ID.
|
||||
|
||||
Args:
|
||||
network_id (str): The ID of the network.
|
||||
verbose (bool): Retrieve the service details across the cluster in
|
||||
swarm mode.
|
||||
scope (str): Filter the network by scope (``swarm``, ``global``
|
||||
or ``local``).
|
||||
|
||||
Returns:
|
||||
(:py:class:`Network`) The network.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.NotFound`
|
||||
If the network does not exist.
|
||||
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
"""
|
||||
return self.prepare_model(
|
||||
self.client.api.inspect_network(network_id, *args, **kwargs)
|
||||
)
|
||||
|
||||
def list(self, *args, **kwargs):
|
||||
"""
|
||||
List networks. Similar to the ``docker network ls`` command.
|
||||
|
||||
Args:
|
||||
names (:py:class:`list`): List of names to filter by.
|
||||
ids (:py:class:`list`): List of ids to filter by.
|
||||
filters (dict): Filters to be processed on the network list.
|
||||
Available filters:
|
||||
- ``driver=[<driver-name>]`` Matches a network's driver.
|
||||
- `label` (str|list): format either ``"key"``, ``"key=value"``
|
||||
or a list of such.
|
||||
- ``type=["custom"|"builtin"]`` Filters networks by type.
|
||||
greedy (bool): Fetch more details for each network individually.
|
||||
You might want this to get the containers attached to them.
|
||||
|
||||
Returns:
|
||||
(list of :py:class:`Network`) The networks on the server.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
greedy = kwargs.pop('greedy', False)
|
||||
resp = self.client.api.networks(*args, **kwargs)
|
||||
networks = [self.prepare_model(item) for item in resp]
|
||||
if greedy and version_gte(self.client.api._version, '1.28'):
|
||||
for net in networks:
|
||||
net.reload()
|
||||
return networks
|
||||
|
||||
def prune(self, filters=None):
|
||||
return self.client.api.prune_networks(filters=filters)
|
||||
prune.__doc__ = APIClient.prune_networks.__doc__
|
||||
@@ -0,0 +1,107 @@
|
||||
from .resource import Collection, Model
|
||||
|
||||
|
||||
class Node(Model):
|
||||
"""A node in a swarm."""
|
||||
id_attribute = 'ID'
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
"""
|
||||
The version number of the service. If this is not the same as the
|
||||
server, the :py:meth:`update` function will not work and you will
|
||||
need to call :py:meth:`reload` before calling it again.
|
||||
"""
|
||||
return self.attrs.get('Version').get('Index')
|
||||
|
||||
def update(self, node_spec):
|
||||
"""
|
||||
Update the node's configuration.
|
||||
|
||||
Args:
|
||||
node_spec (dict): Configuration settings to update. Any values
|
||||
not provided will be removed. Default: ``None``
|
||||
|
||||
Returns:
|
||||
`True` if the request went through.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> node_spec = {'Availability': 'active',
|
||||
'Name': 'node-name',
|
||||
'Role': 'manager',
|
||||
'Labels': {'foo': 'bar'}
|
||||
}
|
||||
>>> node.update(node_spec)
|
||||
|
||||
"""
|
||||
return self.client.api.update_node(self.id, self.version, node_spec)
|
||||
|
||||
def remove(self, force=False):
|
||||
"""
|
||||
Remove this node from the swarm.
|
||||
|
||||
Args:
|
||||
force (bool): Force remove an active node. Default: `False`
|
||||
|
||||
Returns:
|
||||
`True` if the request was successful.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.NotFound`
|
||||
If the node doesn't exist in the swarm.
|
||||
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.client.api.remove_node(self.id, force=force)
|
||||
|
||||
|
||||
class NodeCollection(Collection):
|
||||
"""Nodes on the Docker server."""
|
||||
model = Node
|
||||
|
||||
def get(self, node_id):
|
||||
"""
|
||||
Get a node.
|
||||
|
||||
Args:
|
||||
node_id (string): ID of the node to be inspected.
|
||||
|
||||
Returns:
|
||||
A :py:class:`Node` object.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.prepare_model(self.client.api.inspect_node(node_id))
|
||||
|
||||
def list(self, *args, **kwargs):
|
||||
"""
|
||||
List swarm nodes.
|
||||
|
||||
Args:
|
||||
filters (dict): Filters to process on the nodes list. Valid
|
||||
filters: ``id``, ``name``, ``membership`` and ``role``.
|
||||
Default: ``None``
|
||||
|
||||
Returns:
|
||||
A list of :py:class:`Node` objects.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> client.nodes.list(filters={'role': 'manager'})
|
||||
"""
|
||||
return [
|
||||
self.prepare_model(n)
|
||||
for n in self.client.api.nodes(*args, **kwargs)
|
||||
]
|
||||
@@ -0,0 +1,206 @@
|
||||
from .. import errors
|
||||
from .resource import Collection, Model
|
||||
|
||||
|
||||
class Plugin(Model):
|
||||
"""
|
||||
A plugin on the server.
|
||||
"""
|
||||
def __repr__(self):
|
||||
return f"<{self.__class__.__name__}: '{self.name}'>"
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
"""
|
||||
The plugin's name.
|
||||
"""
|
||||
return self.attrs.get('Name')
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
"""
|
||||
Whether the plugin is enabled.
|
||||
"""
|
||||
return self.attrs.get('Enabled')
|
||||
|
||||
@property
|
||||
def settings(self):
|
||||
"""
|
||||
A dictionary representing the plugin's configuration.
|
||||
"""
|
||||
return self.attrs.get('Settings')
|
||||
|
||||
def configure(self, options):
|
||||
"""
|
||||
Update the plugin's settings.
|
||||
|
||||
Args:
|
||||
options (dict): A key-value mapping of options.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
self.client.api.configure_plugin(self.name, options)
|
||||
self.reload()
|
||||
|
||||
def disable(self, force=False):
|
||||
"""
|
||||
Disable the plugin.
|
||||
|
||||
Args:
|
||||
force (bool): Force disable. Default: False
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
|
||||
self.client.api.disable_plugin(self.name, force)
|
||||
self.reload()
|
||||
|
||||
def enable(self, timeout=0):
|
||||
"""
|
||||
Enable the plugin.
|
||||
|
||||
Args:
|
||||
timeout (int): Timeout in seconds. Default: 0
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
self.client.api.enable_plugin(self.name, timeout)
|
||||
self.reload()
|
||||
|
||||
def push(self):
|
||||
"""
|
||||
Push the plugin to a remote registry.
|
||||
|
||||
Returns:
|
||||
A dict iterator streaming the status of the upload.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.client.api.push_plugin(self.name)
|
||||
|
||||
def remove(self, force=False):
|
||||
"""
|
||||
Remove the plugin from the server.
|
||||
|
||||
Args:
|
||||
force (bool): Remove even if the plugin is enabled.
|
||||
Default: False
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.client.api.remove_plugin(self.name, force=force)
|
||||
|
||||
def upgrade(self, remote=None):
|
||||
"""
|
||||
Upgrade the plugin.
|
||||
|
||||
Args:
|
||||
remote (string): Remote reference to upgrade to. The
|
||||
``:latest`` tag is optional and is the default if omitted.
|
||||
Default: this plugin's name.
|
||||
|
||||
Returns:
|
||||
A generator streaming the decoded API logs
|
||||
"""
|
||||
if self.enabled:
|
||||
raise errors.DockerError(
|
||||
'Plugin must be disabled before upgrading.'
|
||||
)
|
||||
|
||||
if remote is None:
|
||||
remote = self.name
|
||||
privileges = self.client.api.plugin_privileges(remote)
|
||||
yield from self.client.api.upgrade_plugin(
|
||||
self.name,
|
||||
remote,
|
||||
privileges,
|
||||
)
|
||||
self.reload()
|
||||
|
||||
|
||||
class PluginCollection(Collection):
|
||||
model = Plugin
|
||||
|
||||
def create(self, name, plugin_data_dir, gzip=False):
|
||||
"""
|
||||
Create a new plugin.
|
||||
|
||||
Args:
|
||||
name (string): The name of the plugin. The ``:latest`` tag is
|
||||
optional, and is the default if omitted.
|
||||
plugin_data_dir (string): Path to the plugin data directory.
|
||||
Plugin data directory must contain the ``config.json``
|
||||
manifest file and the ``rootfs`` directory.
|
||||
gzip (bool): Compress the context using gzip. Default: False
|
||||
|
||||
Returns:
|
||||
(:py:class:`Plugin`): The newly created plugin.
|
||||
"""
|
||||
self.client.api.create_plugin(name, plugin_data_dir, gzip)
|
||||
return self.get(name)
|
||||
|
||||
def get(self, name):
|
||||
"""
|
||||
Gets a plugin.
|
||||
|
||||
Args:
|
||||
name (str): The name of the plugin.
|
||||
|
||||
Returns:
|
||||
(:py:class:`Plugin`): The plugin.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.NotFound` If the plugin does not
|
||||
exist.
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.prepare_model(self.client.api.inspect_plugin(name))
|
||||
|
||||
def install(self, remote_name, local_name=None):
|
||||
"""
|
||||
Pull and install a plugin.
|
||||
|
||||
Args:
|
||||
remote_name (string): Remote reference for the plugin to
|
||||
install. The ``:latest`` tag is optional, and is the
|
||||
default if omitted.
|
||||
local_name (string): Local name for the pulled plugin.
|
||||
The ``:latest`` tag is optional, and is the default if
|
||||
omitted. Optional.
|
||||
|
||||
Returns:
|
||||
(:py:class:`Plugin`): The installed plugin
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
privileges = self.client.api.plugin_privileges(remote_name)
|
||||
it = self.client.api.pull_plugin(remote_name, privileges, local_name)
|
||||
for _data in it:
|
||||
pass
|
||||
return self.get(local_name or remote_name)
|
||||
|
||||
def list(self):
|
||||
"""
|
||||
List plugins installed on the server.
|
||||
|
||||
Returns:
|
||||
(list of :py:class:`Plugin`): The plugins.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
resp = self.client.api.plugins()
|
||||
return [self.prepare_model(r) for r in resp]
|
||||
@@ -0,0 +1,92 @@
|
||||
class Model:
|
||||
"""
|
||||
A base class for representing a single object on the server.
|
||||
"""
|
||||
id_attribute = 'Id'
|
||||
|
||||
def __init__(self, attrs=None, client=None, collection=None):
|
||||
#: A client pointing at the server that this object is on.
|
||||
self.client = client
|
||||
|
||||
#: The collection that this model is part of.
|
||||
self.collection = collection
|
||||
|
||||
#: The raw representation of this object from the API
|
||||
self.attrs = attrs
|
||||
if self.attrs is None:
|
||||
self.attrs = {}
|
||||
|
||||
def __repr__(self):
|
||||
return f"<{self.__class__.__name__}: {self.short_id}>"
|
||||
|
||||
def __eq__(self, other):
|
||||
return isinstance(other, self.__class__) and self.id == other.id
|
||||
|
||||
def __hash__(self):
|
||||
return hash(f"{self.__class__.__name__}:{self.id}")
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
"""
|
||||
The ID of the object.
|
||||
"""
|
||||
return self.attrs.get(self.id_attribute)
|
||||
|
||||
@property
|
||||
def short_id(self):
|
||||
"""
|
||||
The ID of the object, truncated to 12 characters.
|
||||
"""
|
||||
return self.id[:12]
|
||||
|
||||
def reload(self):
|
||||
"""
|
||||
Load this object from the server again and update ``attrs`` with the
|
||||
new data.
|
||||
"""
|
||||
new_model = self.collection.get(self.id)
|
||||
self.attrs = new_model.attrs
|
||||
|
||||
|
||||
class Collection:
|
||||
"""
|
||||
A base class for representing all objects of a particular type on the
|
||||
server.
|
||||
"""
|
||||
|
||||
#: The type of object this collection represents, set by subclasses
|
||||
model = None
|
||||
|
||||
def __init__(self, client=None):
|
||||
#: The client pointing at the server that this collection of objects
|
||||
#: is on.
|
||||
self.client = client
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
raise TypeError(
|
||||
f"'{self.__class__.__name__}' object is not callable. "
|
||||
"You might be trying to use the old (pre-2.0) API - "
|
||||
"use docker.APIClient if so."
|
||||
)
|
||||
|
||||
def list(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def get(self, key):
|
||||
raise NotImplementedError
|
||||
|
||||
def create(self, attrs=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def prepare_model(self, attrs):
|
||||
"""
|
||||
Create a model from a set of attributes.
|
||||
"""
|
||||
if isinstance(attrs, Model):
|
||||
attrs.client = self.client
|
||||
attrs.collection = self
|
||||
return attrs
|
||||
elif isinstance(attrs, dict):
|
||||
return self.model(attrs=attrs, client=self.client, collection=self)
|
||||
else:
|
||||
raise Exception(f"Can't create {self.model.__name__} from {attrs}")
|
||||
@@ -0,0 +1,70 @@
|
||||
from ..api import APIClient
|
||||
from .resource import Collection, Model
|
||||
|
||||
|
||||
class Secret(Model):
|
||||
"""A secret."""
|
||||
id_attribute = 'ID'
|
||||
|
||||
def __repr__(self):
|
||||
return f"<{self.__class__.__name__}: '{self.name}'>"
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.attrs['Spec']['Name']
|
||||
|
||||
def remove(self):
|
||||
"""
|
||||
Remove this secret.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If secret failed to remove.
|
||||
"""
|
||||
return self.client.api.remove_secret(self.id)
|
||||
|
||||
|
||||
class SecretCollection(Collection):
|
||||
"""Secrets on the Docker server."""
|
||||
model = Secret
|
||||
|
||||
def create(self, **kwargs):
|
||||
obj = self.client.api.create_secret(**kwargs)
|
||||
obj.setdefault("Spec", {})["Name"] = kwargs.get("name")
|
||||
return self.prepare_model(obj)
|
||||
create.__doc__ = APIClient.create_secret.__doc__
|
||||
|
||||
def get(self, secret_id):
|
||||
"""
|
||||
Get a secret.
|
||||
|
||||
Args:
|
||||
secret_id (str): Secret ID.
|
||||
|
||||
Returns:
|
||||
(:py:class:`Secret`): The secret.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.NotFound`
|
||||
If the secret does not exist.
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.prepare_model(self.client.api.inspect_secret(secret_id))
|
||||
|
||||
def list(self, **kwargs):
|
||||
"""
|
||||
List secrets. Similar to the ``docker secret ls`` command.
|
||||
|
||||
Args:
|
||||
filters (dict): Server-side list filtering options.
|
||||
|
||||
Returns:
|
||||
(list of :py:class:`Secret`): The secrets.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
resp = self.client.api.secrets(**kwargs)
|
||||
return [self.prepare_model(obj) for obj in resp]
|
||||
@@ -0,0 +1,390 @@
|
||||
import copy
|
||||
|
||||
from docker.errors import InvalidArgument, create_unexpected_kwargs_error
|
||||
from docker.types import ContainerSpec, Placement, ServiceMode, TaskTemplate
|
||||
|
||||
from .resource import Collection, Model
|
||||
|
||||
|
||||
class Service(Model):
|
||||
"""A service."""
|
||||
id_attribute = 'ID'
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
"""The service's name."""
|
||||
return self.attrs['Spec']['Name']
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
"""
|
||||
The version number of the service. If this is not the same as the
|
||||
server, the :py:meth:`update` function will not work and you will
|
||||
need to call :py:meth:`reload` before calling it again.
|
||||
"""
|
||||
return self.attrs.get('Version').get('Index')
|
||||
|
||||
def remove(self):
|
||||
"""
|
||||
Stop and remove the service.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.client.api.remove_service(self.id)
|
||||
|
||||
def tasks(self, filters=None):
|
||||
"""
|
||||
List the tasks in this service.
|
||||
|
||||
Args:
|
||||
filters (dict): A map of filters to process on the tasks list.
|
||||
Valid filters: ``id``, ``name``, ``node``,
|
||||
``label``, and ``desired-state``.
|
||||
|
||||
Returns:
|
||||
:py:class:`list`: List of task dictionaries.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
if filters is None:
|
||||
filters = {}
|
||||
filters['service'] = self.id
|
||||
return self.client.api.tasks(filters=filters)
|
||||
|
||||
def update(self, **kwargs):
|
||||
"""
|
||||
Update a service's configuration. Similar to the ``docker service
|
||||
update`` command.
|
||||
|
||||
Takes the same parameters as :py:meth:`~ServiceCollection.create`.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
# Image is required, so if it hasn't been set, use current image
|
||||
if 'image' not in kwargs:
|
||||
spec = self.attrs['Spec']['TaskTemplate']['ContainerSpec']
|
||||
kwargs['image'] = spec['Image']
|
||||
|
||||
if kwargs.get('force_update') is True:
|
||||
task_template = self.attrs['Spec']['TaskTemplate']
|
||||
current_value = int(task_template.get('ForceUpdate', 0))
|
||||
kwargs['force_update'] = current_value + 1
|
||||
|
||||
create_kwargs = _get_create_service_kwargs('update', kwargs)
|
||||
|
||||
return self.client.api.update_service(
|
||||
self.id,
|
||||
self.version,
|
||||
**create_kwargs
|
||||
)
|
||||
|
||||
def logs(self, **kwargs):
|
||||
"""
|
||||
Get log stream for the service.
|
||||
Note: This method works only for services with the ``json-file``
|
||||
or ``journald`` logging drivers.
|
||||
|
||||
Args:
|
||||
details (bool): Show extra details provided to logs.
|
||||
Default: ``False``
|
||||
follow (bool): Keep connection open to read logs as they are
|
||||
sent by the Engine. Default: ``False``
|
||||
stdout (bool): Return logs from ``stdout``. Default: ``False``
|
||||
stderr (bool): Return logs from ``stderr``. Default: ``False``
|
||||
since (int): UNIX timestamp for the logs staring point.
|
||||
Default: 0
|
||||
timestamps (bool): Add timestamps to every log line.
|
||||
tail (string or int): Number of log lines to be returned,
|
||||
counting from the current end of the logs. Specify an
|
||||
integer or ``'all'`` to output all log lines.
|
||||
Default: ``all``
|
||||
|
||||
Returns:
|
||||
generator: Logs for the service.
|
||||
"""
|
||||
is_tty = self.attrs['Spec']['TaskTemplate']['ContainerSpec'].get(
|
||||
'TTY', False
|
||||
)
|
||||
return self.client.api.service_logs(self.id, is_tty=is_tty, **kwargs)
|
||||
|
||||
def scale(self, replicas):
|
||||
"""
|
||||
Scale service container.
|
||||
|
||||
Args:
|
||||
replicas (int): The number of containers that should be running.
|
||||
|
||||
Returns:
|
||||
bool: ``True`` if successful.
|
||||
"""
|
||||
|
||||
if 'Global' in self.attrs['Spec']['Mode'].keys():
|
||||
raise InvalidArgument('Cannot scale a global container')
|
||||
|
||||
service_mode = ServiceMode('replicated', replicas)
|
||||
return self.client.api.update_service(self.id, self.version,
|
||||
mode=service_mode,
|
||||
fetch_current_spec=True)
|
||||
|
||||
def force_update(self):
|
||||
"""
|
||||
Force update the service even if no changes require it.
|
||||
|
||||
Returns:
|
||||
bool: ``True`` if successful.
|
||||
"""
|
||||
|
||||
return self.update(force_update=True, fetch_current_spec=True)
|
||||
|
||||
|
||||
class ServiceCollection(Collection):
|
||||
"""Services on the Docker server."""
|
||||
model = Service
|
||||
|
||||
def create(self, image, command=None, **kwargs):
|
||||
"""
|
||||
Create a service. Similar to the ``docker service create`` command.
|
||||
|
||||
Args:
|
||||
image (str): The image name to use for the containers.
|
||||
command (list of str or str): Command to run.
|
||||
args (list of str): Arguments to the command.
|
||||
constraints (list of str): :py:class:`~docker.types.Placement`
|
||||
constraints.
|
||||
preferences (list of tuple): :py:class:`~docker.types.Placement`
|
||||
preferences.
|
||||
maxreplicas (int): :py:class:`~docker.types.Placement` maxreplicas
|
||||
or (int) representing maximum number of replicas per node.
|
||||
platforms (list of tuple): A list of platform constraints
|
||||
expressed as ``(arch, os)`` tuples.
|
||||
container_labels (dict): Labels to apply to the container.
|
||||
endpoint_spec (EndpointSpec): Properties that can be configured to
|
||||
access and load balance a service. Default: ``None``.
|
||||
env (list of str): Environment variables, in the form
|
||||
``KEY=val``.
|
||||
hostname (string): Hostname to set on the container.
|
||||
init (boolean): Run an init inside the container that forwards
|
||||
signals and reaps processes
|
||||
isolation (string): Isolation technology used by the service's
|
||||
containers. Only used for Windows containers.
|
||||
labels (dict): Labels to apply to the service.
|
||||
log_driver (str): Log driver to use for containers.
|
||||
log_driver_options (dict): Log driver options.
|
||||
mode (ServiceMode): Scheduling mode for the service.
|
||||
Default:``None``
|
||||
mounts (list of str): Mounts for the containers, in the form
|
||||
``source:target:options``, where options is either
|
||||
``ro`` or ``rw``.
|
||||
name (str): Name to give to the service.
|
||||
networks (:py:class:`list`): List of network names or IDs or
|
||||
:py:class:`~docker.types.NetworkAttachmentConfig` to attach the
|
||||
service to. Default: ``None``.
|
||||
resources (Resources): Resource limits and reservations.
|
||||
restart_policy (RestartPolicy): Restart policy for containers.
|
||||
secrets (list of :py:class:`~docker.types.SecretReference`): List
|
||||
of secrets accessible to containers for this service.
|
||||
stop_grace_period (int): Amount of time to wait for
|
||||
containers to terminate before forcefully killing them.
|
||||
update_config (UpdateConfig): Specification for the update strategy
|
||||
of the service. Default: ``None``
|
||||
rollback_config (RollbackConfig): Specification for the rollback
|
||||
strategy of the service. Default: ``None``
|
||||
user (str): User to run commands as.
|
||||
workdir (str): Working directory for commands to run.
|
||||
tty (boolean): Whether a pseudo-TTY should be allocated.
|
||||
groups (:py:class:`list`): A list of additional groups that the
|
||||
container process will run as.
|
||||
open_stdin (boolean): Open ``stdin``
|
||||
read_only (boolean): Mount the container's root filesystem as read
|
||||
only.
|
||||
stop_signal (string): Set signal to stop the service's containers
|
||||
healthcheck (Healthcheck): Healthcheck
|
||||
configuration for this service.
|
||||
hosts (:py:class:`dict`): A set of host to IP mappings to add to
|
||||
the container's `hosts` file.
|
||||
dns_config (DNSConfig): Specification for DNS
|
||||
related configurations in resolver configuration file.
|
||||
configs (:py:class:`list`): List of
|
||||
:py:class:`~docker.types.ConfigReference` that will be exposed
|
||||
to the service.
|
||||
privileges (Privileges): Security options for the service's
|
||||
containers.
|
||||
cap_add (:py:class:`list`): A list of kernel capabilities to add to
|
||||
the default set for the container.
|
||||
cap_drop (:py:class:`list`): A list of kernel capabilities to drop
|
||||
from the default set for the container.
|
||||
sysctls (:py:class:`dict`): A dict of sysctl values to add to the
|
||||
container
|
||||
|
||||
Returns:
|
||||
:py:class:`Service`: The created service.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
kwargs['image'] = image
|
||||
kwargs['command'] = command
|
||||
create_kwargs = _get_create_service_kwargs('create', kwargs)
|
||||
service_id = self.client.api.create_service(**create_kwargs)
|
||||
return self.get(service_id)
|
||||
|
||||
def get(self, service_id, insert_defaults=None):
|
||||
"""
|
||||
Get a service.
|
||||
|
||||
Args:
|
||||
service_id (str): The ID of the service.
|
||||
insert_defaults (boolean): If true, default values will be merged
|
||||
into the output.
|
||||
|
||||
Returns:
|
||||
:py:class:`Service`: The service.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.NotFound`
|
||||
If the service does not exist.
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
:py:class:`docker.errors.InvalidVersion`
|
||||
If one of the arguments is not supported with the current
|
||||
API version.
|
||||
"""
|
||||
return self.prepare_model(
|
||||
self.client.api.inspect_service(service_id, insert_defaults)
|
||||
)
|
||||
|
||||
def list(self, **kwargs):
|
||||
"""
|
||||
List services.
|
||||
|
||||
Args:
|
||||
filters (dict): Filters to process on the nodes list. Valid
|
||||
filters: ``id``, ``name`` , ``label`` and ``mode``.
|
||||
Default: ``None``.
|
||||
status (bool): Include the service task count of running and
|
||||
desired tasks. Default: ``None``.
|
||||
|
||||
Returns:
|
||||
list of :py:class:`Service`: The services.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return [
|
||||
self.prepare_model(s)
|
||||
for s in self.client.api.services(**kwargs)
|
||||
]
|
||||
|
||||
|
||||
# kwargs to copy straight over to ContainerSpec
|
||||
CONTAINER_SPEC_KWARGS = [
|
||||
'args',
|
||||
'cap_add',
|
||||
'cap_drop',
|
||||
'command',
|
||||
'configs',
|
||||
'dns_config',
|
||||
'env',
|
||||
'groups',
|
||||
'healthcheck',
|
||||
'hostname',
|
||||
'hosts',
|
||||
'image',
|
||||
'init',
|
||||
'isolation',
|
||||
'labels',
|
||||
'mounts',
|
||||
'open_stdin',
|
||||
'privileges',
|
||||
'read_only',
|
||||
'secrets',
|
||||
'stop_grace_period',
|
||||
'stop_signal',
|
||||
'tty',
|
||||
'user',
|
||||
'workdir',
|
||||
'sysctls',
|
||||
]
|
||||
|
||||
# kwargs to copy straight over to TaskTemplate
|
||||
TASK_TEMPLATE_KWARGS = [
|
||||
'networks',
|
||||
'resources',
|
||||
'restart_policy',
|
||||
]
|
||||
|
||||
# kwargs to copy straight over to create_service
|
||||
CREATE_SERVICE_KWARGS = [
|
||||
'name',
|
||||
'labels',
|
||||
'mode',
|
||||
'update_config',
|
||||
'rollback_config',
|
||||
'endpoint_spec',
|
||||
]
|
||||
|
||||
PLACEMENT_KWARGS = [
|
||||
'constraints',
|
||||
'preferences',
|
||||
'platforms',
|
||||
'maxreplicas',
|
||||
]
|
||||
|
||||
|
||||
def _get_create_service_kwargs(func_name, kwargs):
|
||||
# Copy over things which can be copied directly
|
||||
create_kwargs = {}
|
||||
for key in copy.copy(kwargs):
|
||||
if key in CREATE_SERVICE_KWARGS:
|
||||
create_kwargs[key] = kwargs.pop(key)
|
||||
container_spec_kwargs = {}
|
||||
for key in copy.copy(kwargs):
|
||||
if key in CONTAINER_SPEC_KWARGS:
|
||||
container_spec_kwargs[key] = kwargs.pop(key)
|
||||
task_template_kwargs = {}
|
||||
for key in copy.copy(kwargs):
|
||||
if key in TASK_TEMPLATE_KWARGS:
|
||||
task_template_kwargs[key] = kwargs.pop(key)
|
||||
|
||||
if 'container_labels' in kwargs:
|
||||
container_spec_kwargs['labels'] = kwargs.pop('container_labels')
|
||||
|
||||
placement = {}
|
||||
for key in copy.copy(kwargs):
|
||||
if key in PLACEMENT_KWARGS:
|
||||
placement[key] = kwargs.pop(key)
|
||||
placement = Placement(**placement)
|
||||
task_template_kwargs['placement'] = placement
|
||||
|
||||
if 'log_driver' in kwargs:
|
||||
task_template_kwargs['log_driver'] = {
|
||||
'Name': kwargs.pop('log_driver'),
|
||||
'Options': kwargs.pop('log_driver_options', {})
|
||||
}
|
||||
|
||||
if func_name == 'update':
|
||||
if 'force_update' in kwargs:
|
||||
task_template_kwargs['force_update'] = kwargs.pop('force_update')
|
||||
|
||||
# fetch the current spec by default if updating the service
|
||||
# through the model
|
||||
fetch_current_spec = kwargs.pop('fetch_current_spec', True)
|
||||
create_kwargs['fetch_current_spec'] = fetch_current_spec
|
||||
|
||||
# All kwargs should have been consumed by this point, so raise
|
||||
# error if any are left
|
||||
if kwargs:
|
||||
raise create_unexpected_kwargs_error(func_name, kwargs)
|
||||
|
||||
container_spec = ContainerSpec(**container_spec_kwargs)
|
||||
task_template_kwargs['container_spec'] = container_spec
|
||||
create_kwargs['task_template'] = TaskTemplate(**task_template_kwargs)
|
||||
return create_kwargs
|
||||
@@ -0,0 +1,190 @@
|
||||
from docker.api import APIClient
|
||||
from docker.errors import APIError
|
||||
|
||||
from .resource import Model
|
||||
|
||||
|
||||
class Swarm(Model):
|
||||
"""
|
||||
The server's Swarm state. This a singleton that must be reloaded to get
|
||||
the current state of the Swarm.
|
||||
"""
|
||||
id_attribute = 'ID'
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
if self.client:
|
||||
try:
|
||||
self.reload()
|
||||
except APIError as e:
|
||||
# FIXME: https://github.com/docker/docker/issues/29192
|
||||
if e.response.status_code not in (406, 503):
|
||||
raise
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
"""
|
||||
The version number of the swarm. If this is not the same as the
|
||||
server, the :py:meth:`update` function will not work and you will
|
||||
need to call :py:meth:`reload` before calling it again.
|
||||
"""
|
||||
return self.attrs.get('Version').get('Index')
|
||||
|
||||
def get_unlock_key(self):
|
||||
return self.client.api.get_unlock_key()
|
||||
get_unlock_key.__doc__ = APIClient.get_unlock_key.__doc__
|
||||
|
||||
def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377',
|
||||
force_new_cluster=False, default_addr_pool=None,
|
||||
subnet_size=None, data_path_addr=None, data_path_port=None,
|
||||
**kwargs):
|
||||
"""
|
||||
Initialize a new swarm on this Engine.
|
||||
|
||||
Args:
|
||||
advertise_addr (str): Externally reachable address advertised to
|
||||
other nodes. This can either be an address/port combination in
|
||||
the form ``192.168.1.1:4567``, or an interface followed by a
|
||||
port number, like ``eth0:4567``. If the port number is omitted,
|
||||
the port number from the listen address is used.
|
||||
|
||||
If not specified, it will be automatically detected when
|
||||
possible.
|
||||
listen_addr (str): Listen address used for inter-manager
|
||||
communication, as well as determining the networking interface
|
||||
used for the VXLAN Tunnel Endpoint (VTEP). This can either be
|
||||
an address/port combination in the form ``192.168.1.1:4567``,
|
||||
or an interface followed by a port number, like ``eth0:4567``.
|
||||
If the port number is omitted, the default swarm listening port
|
||||
is used. Default: ``0.0.0.0:2377``
|
||||
force_new_cluster (bool): Force creating a new Swarm, even if
|
||||
already part of one. Default: False
|
||||
default_addr_pool (list of str): Default Address Pool specifies
|
||||
default subnet pools for global scope networks. Each pool
|
||||
should be specified as a CIDR block, like '10.0.0.0/8'.
|
||||
Default: None
|
||||
subnet_size (int): SubnetSize specifies the subnet size of the
|
||||
networks created from the default subnet pool. Default: None
|
||||
data_path_addr (string): Address or interface to use for data path
|
||||
traffic. For example, 192.168.1.1, or an interface, like eth0.
|
||||
data_path_port (int): Port number to use for data path traffic.
|
||||
Acceptable port range is 1024 to 49151. If set to ``None`` or
|
||||
0, the default port 4789 will be used. Default: None
|
||||
task_history_retention_limit (int): Maximum number of tasks
|
||||
history stored.
|
||||
snapshot_interval (int): Number of logs entries between snapshot.
|
||||
keep_old_snapshots (int): Number of snapshots to keep beyond the
|
||||
current snapshot.
|
||||
log_entries_for_slow_followers (int): Number of log entries to
|
||||
keep around to sync up slow followers after a snapshot is
|
||||
created.
|
||||
heartbeat_tick (int): Amount of ticks (in seconds) between each
|
||||
heartbeat.
|
||||
election_tick (int): Amount of ticks (in seconds) needed without a
|
||||
leader to trigger a new election.
|
||||
dispatcher_heartbeat_period (int): The delay for an agent to send
|
||||
a heartbeat to the dispatcher.
|
||||
node_cert_expiry (int): Automatic expiry for nodes certificates.
|
||||
external_ca (dict): Configuration for forwarding signing requests
|
||||
to an external certificate authority. Use
|
||||
``docker.types.SwarmExternalCA``.
|
||||
name (string): Swarm's name
|
||||
labels (dict): User-defined key/value metadata.
|
||||
signing_ca_cert (str): The desired signing CA certificate for all
|
||||
swarm node TLS leaf certificates, in PEM format.
|
||||
signing_ca_key (str): The desired signing CA key for all swarm
|
||||
node TLS leaf certificates, in PEM format.
|
||||
ca_force_rotate (int): An integer whose purpose is to force swarm
|
||||
to generate a new signing CA certificate and key, if none have
|
||||
been specified.
|
||||
autolock_managers (boolean): If set, generate a key and use it to
|
||||
lock data stored on the managers.
|
||||
log_driver (DriverConfig): The default log driver to use for tasks
|
||||
created in the orchestrator.
|
||||
|
||||
Returns:
|
||||
(str): The ID of the created node.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> client.swarm.init(
|
||||
advertise_addr='eth0', listen_addr='0.0.0.0:5000',
|
||||
force_new_cluster=False, default_addr_pool=['10.20.0.0/16],
|
||||
subnet_size=24, snapshot_interval=5000,
|
||||
log_entries_for_slow_followers=1200
|
||||
)
|
||||
|
||||
"""
|
||||
init_kwargs = {
|
||||
'advertise_addr': advertise_addr,
|
||||
'listen_addr': listen_addr,
|
||||
'force_new_cluster': force_new_cluster,
|
||||
'default_addr_pool': default_addr_pool,
|
||||
'subnet_size': subnet_size,
|
||||
'data_path_addr': data_path_addr,
|
||||
'data_path_port': data_path_port,
|
||||
}
|
||||
init_kwargs['swarm_spec'] = self.client.api.create_swarm_spec(**kwargs)
|
||||
node_id = self.client.api.init_swarm(**init_kwargs)
|
||||
self.reload()
|
||||
return node_id
|
||||
|
||||
def join(self, *args, **kwargs):
|
||||
return self.client.api.join_swarm(*args, **kwargs)
|
||||
join.__doc__ = APIClient.join_swarm.__doc__
|
||||
|
||||
def leave(self, *args, **kwargs):
|
||||
return self.client.api.leave_swarm(*args, **kwargs)
|
||||
leave.__doc__ = APIClient.leave_swarm.__doc__
|
||||
|
||||
def reload(self):
|
||||
"""
|
||||
Inspect the swarm on the server and store the response in
|
||||
:py:attr:`attrs`.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
self.attrs = self.client.api.inspect_swarm()
|
||||
|
||||
def unlock(self, key):
|
||||
return self.client.api.unlock_swarm(key)
|
||||
unlock.__doc__ = APIClient.unlock_swarm.__doc__
|
||||
|
||||
def update(self, rotate_worker_token=False, rotate_manager_token=False,
|
||||
rotate_manager_unlock_key=False, **kwargs):
|
||||
"""
|
||||
Update the swarm's configuration.
|
||||
|
||||
It takes the same arguments as :py:meth:`init`, except
|
||||
``advertise_addr``, ``listen_addr``, and ``force_new_cluster``. In
|
||||
addition, it takes these arguments:
|
||||
|
||||
Args:
|
||||
rotate_worker_token (bool): Rotate the worker join token. Default:
|
||||
``False``.
|
||||
rotate_manager_token (bool): Rotate the manager join token.
|
||||
Default: ``False``.
|
||||
rotate_manager_unlock_key (bool): Rotate the manager unlock key.
|
||||
Default: ``False``.
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
"""
|
||||
# this seems to have to be set
|
||||
if kwargs.get('node_cert_expiry') is None:
|
||||
kwargs['node_cert_expiry'] = 7776000000000000
|
||||
|
||||
return self.client.api.update_swarm(
|
||||
version=self.version,
|
||||
swarm_spec=self.client.api.create_swarm_spec(**kwargs),
|
||||
rotate_worker_token=rotate_worker_token,
|
||||
rotate_manager_token=rotate_manager_token,
|
||||
rotate_manager_unlock_key=rotate_manager_unlock_key
|
||||
)
|
||||
@@ -0,0 +1,99 @@
|
||||
from ..api import APIClient
|
||||
from .resource import Collection, Model
|
||||
|
||||
|
||||
class Volume(Model):
|
||||
"""A volume."""
|
||||
id_attribute = 'Name'
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
"""The name of the volume."""
|
||||
return self.attrs['Name']
|
||||
|
||||
def remove(self, force=False):
|
||||
"""
|
||||
Remove this volume.
|
||||
|
||||
Args:
|
||||
force (bool): Force removal of volumes that were already removed
|
||||
out of band by the volume driver plugin.
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If volume failed to remove.
|
||||
"""
|
||||
return self.client.api.remove_volume(self.id, force=force)
|
||||
|
||||
|
||||
class VolumeCollection(Collection):
|
||||
"""Volumes on the Docker server."""
|
||||
model = Volume
|
||||
|
||||
def create(self, name=None, **kwargs):
|
||||
"""
|
||||
Create a volume.
|
||||
|
||||
Args:
|
||||
name (str): Name of the volume. If not specified, the engine
|
||||
generates a name.
|
||||
driver (str): Name of the driver used to create the volume
|
||||
driver_opts (dict): Driver options as a key-value dictionary
|
||||
labels (dict): Labels to set on the volume
|
||||
|
||||
Returns:
|
||||
(:py:class:`Volume`): The volume created.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
|
||||
Example:
|
||||
|
||||
>>> volume = client.volumes.create(name='foobar', driver='local',
|
||||
driver_opts={'foo': 'bar', 'baz': 'false'},
|
||||
labels={"key": "value"})
|
||||
|
||||
"""
|
||||
obj = self.client.api.create_volume(name, **kwargs)
|
||||
return self.prepare_model(obj)
|
||||
|
||||
def get(self, volume_id):
|
||||
"""
|
||||
Get a volume.
|
||||
|
||||
Args:
|
||||
volume_id (str): Volume name.
|
||||
|
||||
Returns:
|
||||
(:py:class:`Volume`): The volume.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.NotFound`
|
||||
If the volume does not exist.
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
return self.prepare_model(self.client.api.inspect_volume(volume_id))
|
||||
|
||||
def list(self, **kwargs):
|
||||
"""
|
||||
List volumes. Similar to the ``docker volume ls`` command.
|
||||
|
||||
Args:
|
||||
filters (dict): Server-side list filtering options.
|
||||
|
||||
Returns:
|
||||
(list of :py:class:`Volume`): The volumes.
|
||||
|
||||
Raises:
|
||||
:py:class:`docker.errors.APIError`
|
||||
If the server returns an error.
|
||||
"""
|
||||
resp = self.client.api.volumes(**kwargs)
|
||||
if not resp.get('Volumes'):
|
||||
return []
|
||||
return [self.prepare_model(obj) for obj in resp['Volumes']]
|
||||
|
||||
def prune(self, filters=None):
|
||||
return self.client.api.prune_volumes(filters=filters)
|
||||
prune.__doc__ = APIClient.prune_volumes.__doc__
|
||||
@@ -0,0 +1,67 @@
|
||||
import os
|
||||
|
||||
from . import errors
|
||||
|
||||
|
||||
class TLSConfig:
|
||||
"""
|
||||
TLS configuration.
|
||||
|
||||
Args:
|
||||
client_cert (tuple of str): Path to client cert, path to client key.
|
||||
ca_cert (str): Path to CA cert file.
|
||||
verify (bool or str): This can be a bool or a path to a CA cert
|
||||
file to verify against. If ``True``, verify using ca_cert;
|
||||
if ``False`` or not specified, do not verify.
|
||||
"""
|
||||
cert = None
|
||||
ca_cert = None
|
||||
verify = None
|
||||
|
||||
def __init__(self, client_cert=None, ca_cert=None, verify=None):
|
||||
# Argument compatibility/mapping with
|
||||
# https://docs.docker.com/engine/articles/https/
|
||||
# This diverges from the Docker CLI in that users can specify 'tls'
|
||||
# here, but also disable any public/default CA pool verification by
|
||||
# leaving verify=False
|
||||
|
||||
# "client_cert" must have both or neither cert/key files. In
|
||||
# either case, Alert the user when both are expected, but any are
|
||||
# missing.
|
||||
|
||||
if client_cert:
|
||||
try:
|
||||
tls_cert, tls_key = client_cert
|
||||
except ValueError:
|
||||
raise errors.TLSParameterError(
|
||||
'client_cert must be a tuple of'
|
||||
' (client certificate, key file)'
|
||||
) from None
|
||||
|
||||
if not (tls_cert and tls_key) or (not os.path.isfile(tls_cert) or
|
||||
not os.path.isfile(tls_key)):
|
||||
raise errors.TLSParameterError(
|
||||
'Path to a certificate and key files must be provided'
|
||||
' through the client_cert param'
|
||||
)
|
||||
self.cert = (tls_cert, tls_key)
|
||||
|
||||
# If verify is set, make sure the cert exists
|
||||
self.verify = verify
|
||||
self.ca_cert = ca_cert
|
||||
if self.verify and self.ca_cert and not os.path.isfile(self.ca_cert):
|
||||
raise errors.TLSParameterError(
|
||||
'Invalid CA certificate provided for `ca_cert`.'
|
||||
)
|
||||
|
||||
def configure_client(self, client):
|
||||
"""
|
||||
Configure a client with these TLS options.
|
||||
"""
|
||||
if self.verify and self.ca_cert:
|
||||
client.verify = self.ca_cert
|
||||
else:
|
||||
client.verify = self.verify
|
||||
|
||||
if self.cert:
|
||||
client.cert = self.cert
|
||||
@@ -0,0 +1,12 @@
|
||||
from .unixconn import UnixHTTPAdapter
|
||||
|
||||
try:
|
||||
from .npipeconn import NpipeHTTPAdapter
|
||||
from .npipesocket import NpipeSocket
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from .sshconn import SSHHTTPAdapter
|
||||
except ImportError:
|
||||
pass
|
||||
@@ -0,0 +1,13 @@
|
||||
import requests.adapters
|
||||
|
||||
|
||||
class BaseHTTPAdapter(requests.adapters.HTTPAdapter):
|
||||
def close(self):
|
||||
super().close()
|
||||
if hasattr(self, 'pools'):
|
||||
self.pools.clear()
|
||||
|
||||
# Fix for requests 2.32.2+:
|
||||
# https://github.com/psf/requests/commit/c98e4d133ef29c46a9b68cd783087218a8075e05
|
||||
def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
|
||||
return self.get_connection(request.url, proxies)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user