Skip to content

Preserve individual volumes on recreate#858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aanand merged 1 commit into from
May 14, 2015
+152 −40
Merged

Preserve individual volumes on recreate #858

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions compose/cli/main.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -317,10 +317,7 @@ def run(self, project, options):
}

if options['-e']:
# Merge environment from config with -e command line
container_options['environment'] = dict(
parse_environment(service.options.get('environment')),
**parse_environment(options['-e']))
container_options['environment'] = parse_environment(options['-e'])

if options['--entrypoint']:
container_options['entrypoint'] = options.get('--entrypoint')
Expand Down
4 changes: 4 additions & 0 deletions compose/container.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -44,6 +44,10 @@ def id(self):
def image(self):
return self.dictionary['Image']

@property
def image_config(self):
return self.client.inspect_image(self.image)

@property
def short_id(self):
return self.id[:10]
Expand Down
86 changes: 70 additions & 16 deletions compose/service.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,14 +3,14 @@
from collections import namedtuple
import logging
import re
from operator import attrgetter
import sys
import six
from operator import attrgetter

import six
from docker.errors import APIError
from docker.utils import create_host_config, LogConfig

from .config import DOCKER_CONFIG_KEYS
from .config import DOCKER_CONFIG_KEYS, merge_environment
from .container import Container, get_container_name
from .progress_stream import stream_output, StreamOutputError

Expand DownExpand Up@@ -183,10 +183,10 @@ def create_container(self,
Create a container for this service. If the image doesn't exist, attempt to pull
it.
"""
override_options['volumes_from'] = self._get_volumes_from(previous_container)
container_options = self._get_container_create_options(
override_options,
one_off=one_off,
previous_container=previous_container,
)

if (do_build and
Expand DownExpand Up@@ -247,6 +247,12 @@ def recreate_container(self, container, **override_options):
self.client.rename(
container.id,
'%s_%s' % (container.short_id, container.name))

override_options = dict(
override_options,
environment=merge_environment(
override_options.get('environment'),
{'affinity:container': '=' + container.id}))
new_container = self.create_container(
do_build=False,
previous_container=container,
Expand DownExpand Up@@ -326,7 +332,7 @@ def _get_links(self, link_to_self):
links.append((external_link, link_name))
return links

def _get_volumes_from(self, previous_container=None):
def _get_volumes_from(self):
volumes_from = []
for volume_source in self.volumes_from:
if isinstance(volume_source, Service):
Expand All@@ -339,9 +345,6 @@ def _get_volumes_from(self, previous_container=None):
elif isinstance(volume_source, Container):
volumes_from.append(volume_source.id)

if previous_container:
volumes_from.append(previous_container.id)

return volumes_from

def _get_net(self):
Expand All@@ -363,7 +366,11 @@ def _get_net(self):

return net

def _get_container_create_options(self, override_options, one_off=False):
def _get_container_create_options(
self,
override_options,
one_off=False,
previous_container=None):
container_options = dict(
(k, self.options[k])
for k in DOCKER_CONFIG_KEYS if k in self.options)
Expand DownExpand Up@@ -396,11 +403,19 @@ def _get_container_create_options(self, override_options, one_off=False):
ports.append(port)
container_options['ports'] = ports

override_options['binds'] = merge_volume_bindings(
container_options.get('volumes') or [],
previous_container)

if 'volumes' in container_options:
container_options['volumes'] = dict(
(parse_volume_spec(v).internal, {})
for v in container_options['volumes'])

container_options['environment'] = merge_environment(
self.options.get('environment'),
override_options.get('environment'))

if self.can_be_built():
container_options['image'] = self.full_name

Expand All@@ -418,11 +433,6 @@ def _get_container_host_config(self, override_options, one_off=False):
options = dict(self.options, **override_options)
port_bindings = build_port_bindings(options.get('ports') or [])

volume_bindings = dict(
build_volume_binding(parse_volume_spec(volume))
for volume in options.get('volumes') or []
if ':' in volume)

privileged = options.get('privileged', False)
cap_add = options.get('cap_add', None)
cap_drop = options.get('cap_drop', None)
Expand All@@ -447,8 +457,8 @@ def _get_container_host_config(self, override_options, one_off=False):
return create_host_config(
links=self._get_links(link_to_self=one_off),
port_bindings=port_bindings,
binds=volume_bindings,
volumes_from=options.get('volumes_from'),
binds=options.get('binds'),
volumes_from=self._get_volumes_from(),
privileged=privileged,
network_mode=self._get_net(),
devices=devices,
Expand DownExpand Up@@ -531,6 +541,50 @@ def pull(self, insecure_registry=False):
stream_output(output, sys.stdout)


def get_container_data_volumes(container, volumes_option):
"""Find the container data volumes that are in `volumes_option`, and return
a mapping of volume bindings for those volumes.
"""
volumes = []

volumes_option = volumes_option or []
container_volumes = container.get('Volumes') or {}
image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {}

for volume in set(volumes_option + image_volumes.keys()):
volume = parse_volume_spec(volume)
# No need to preserve host volumes
if volume.external:
continue

volume_path = container_volumes.get(volume.internal)
# New volume, doesn't exist in the old container
if not volume_path:
continue

# Copy existing volume from old container
volume = volume._replace(external=volume_path)
volumes.append(build_volume_binding(volume))

return dict(volumes)


def merge_volume_bindings(volumes_option, previous_container):
"""Return a list of volume bindings for a container. Container data volumes
are replaced by those from the previous container.
"""
volume_bindings = dict(
build_volume_binding(parse_volume_spec(volume))
for volume in volumes_option or []
if ':' in volume)

if previous_container:
volume_bindings.update(
get_container_data_volumes(previous_container, volumes_option))

return volume_bindings


NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')


Expand Down
27 changes: 14 additions & 13 deletions tests/integration/service_test.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def test_create_container_with_unspecified_volume(self):
service = self.create_service('db', volumes=['/var/db'])
container = service.create_container()
service.start_container(container)
self.assertIn('/var/db', container.inspect()['Volumes'])
self.assertIn('/var/db', container.get('Volumes'))

def test_create_container_with_cpu_shares(self):
service = self.create_service('db', cpu_shares=73)
Expand DownExpand Up@@ -239,24 +239,27 @@ def test_recreate_containers(self):
command=['300']
)
old_container = service.create_container()
self.assertEqual(old_container.dictionary['Config']['Entrypoint'], ['sleep'])
self.assertEqual(old_container.dictionary['Config']['Cmd'], ['300'])
self.assertIn('FOO=1', old_container.dictionary['Config']['Env'])
self.assertEqual(old_container.get('Config.Entrypoint'), ['sleep'])
self.assertEqual(old_container.get('Config.Cmd'), ['300'])
self.assertIn('FOO=1', old_container.get('Config.Env'))
self.assertEqual(old_container.name, 'composetest_db_1')
service.start_container(old_container)
volume_path = old_container.inspect()['Volumes']['/etc']
old_container.inspect() # reload volume data
volume_path = old_container.get('Volumes')['/etc']

num_containers_before = len(self.client.containers(all=True))

service.options['environment']['FOO'] = '2'
new_container, = service.recreate_containers()

self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['sleep'])
self.assertEqual(new_container.dictionary['Config']['Cmd'], ['300'])
self.assertIn('FOO=2', new_container.dictionary['Config']['Env'])
self.assertEqual(new_container.get('Config.Entrypoint'), ['sleep'])
self.assertEqual(new_container.get('Config.Cmd'), ['300'])
self.assertIn('FOO=2', new_container.get('Config.Env'))
self.assertEqual(new_container.name, 'composetest_db_1')
self.assertEqual(new_container.inspect()['Volumes']['/etc'], volume_path)
self.assertIn(old_container.id, new_container.dictionary['HostConfig']['VolumesFrom'])
self.assertEqual(new_container.get('Volumes')['/etc'], volume_path)
self.assertIn(
'affinity:container==%s' % old_container.id,
new_container.get('Config.Env'))

self.assertEqual(len(self.client.containers(all=True)), num_containers_before)
self.assertNotEqual(old_container.id, new_container.id)
Expand DownExpand Up@@ -289,9 +292,7 @@ def test_recreate_containers_with_image_declared_volume(self):
self.assertEqual(old_container.get('Volumes').keys(), ['/data'])
volume_path = old_container.get('Volumes')['/data']

service.recreate_containers()
new_container = service.containers()[0]
service.start_container(new_container)
new_container = service.recreate_containers()[0]
self.assertEqual(new_container.get('Volumes').keys(), ['/data'])
self.assertEqual(new_container.get('Volumes')['/data'], volume_path)

Expand Down
70 changes: 63 additions & 7 deletions tests/unit/service_test.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,9 @@
ConfigError,
build_port_bindings,
build_volume_binding,
get_container_data_volumes,
get_container_name,
merge_volume_bindings,
parse_repository_tag,
parse_volume_spec,
split_port,
Expand DownExpand Up@@ -86,13 +88,6 @@ def test_get_volumes_from_container(self):

self.assertEqual(service._get_volumes_from(), [container_id])

def test_get_volumes_from_previous_container(self):
container_id = 'aabbccddee'
service = Service('test', image='foo')
container = mock.Mock(id=container_id, spec=Container, image='foo')

self.assertEqual(service._get_volumes_from(container), [container_id])

def test_get_volumes_from_service_container_exists(self):
container_ids = ['aabbccddee', '12345']
from_service = mock.create_autospec(Service)
Expand DownExpand Up@@ -320,6 +315,9 @@ def test_create_container_no_build(self):

class ServiceVolumesTest(unittest.TestCase):

def setUp(self):
self.mock_client = mock.create_autospec(docker.Client)

def test_parse_volume_spec_only_one_path(self):
spec = parse_volume_spec('/the/volume')
self.assertEqual(spec, (None, '/the/volume', 'rw'))
Expand All@@ -345,3 +343,61 @@ def test_build_volume_binding(self):
self.assertEqual(
binding,
('/outside', dict(bind='/inside', ro=False)))

def test_get_container_data_volumes(self):
options = [
'/host/volume:/host/volume:ro',
'/new/volume',
'/existing/volume',
]

self.mock_client.inspect_image.return_value = {
'ContainerConfig': {
'Volumes': {
'/mnt/image/data': {},
}
}
}
container = Container(self.mock_client, {
'Image': 'ababab',
'Volumes': {
'/host/volume': '/host/volume',
'/existing/volume': '/var/lib/docker/aaaaaaaa',
'/removed/volume': '/var/lib/docker/bbbbbbbb',
'/mnt/image/data': '/var/lib/docker/cccccccc',
},
}, has_been_inspected=True)

expected = {
'/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
'/var/lib/docker/cccccccc': {'bind': '/mnt/image/data', 'ro': False},
}

binds = get_container_data_volumes(container, options)
self.assertEqual(binds, expected)

def test_merge_volume_bindings(self):
options = [
'/host/volume:/host/volume:ro',
'/host/rw/volume:/host/rw/volume',
'/new/volume',
'/existing/volume',
]

self.mock_client.inspect_image.return_value = {
'ContainerConfig': {'Volumes': {}}
}

intermediate_container = Container(self.mock_client, {
'Image': 'ababab',
'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'},
}, has_been_inspected=True)

expected = {
'/host/volume': {'bind': '/host/volume', 'ro': True},
'/host/rw/volume': {'bind': '/host/rw/volume', 'ro': False},
'/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
}

binds = merge_volume_bindings(options, intermediate_container)
self.assertEqual(binds, expected)
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Preserve individual volumes on recreate by dnephin · Pull Request #858 · docker/compose · GitHub
Skip to content
Preserve individual volumes on recreate#858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

aanand
merged 1 commit into

from

May 14, 2015

+152

−40

 Merged


Preserve individual volumes on recreate
#858
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5 changes: 1 addition & 4 deletions
compose/cli/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,10 +317,7 @@ def run(self, project, options):
 }

 if options['-e']:
 # Merge environment from config with -e command line
 container_options['environment'] = dict(
 parse_environment(service.options.get('environment')),
 **parse_environment(options['-e']))
 container_options['environment'] = parse_environment(options['-e'])

 if options['--entrypoint']:
 container_options['entrypoint'] = options.get('--entrypoint')
Expand Down

4 changes: 4 additions & 0 deletions
compose/container.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -44,6 +44,10 @@ def id(self):
 def image(self):
 return self.dictionary['Image']

 @property
 def image_config(self):
 return self.client.inspect_image(self.image)

 @property
 def short_id(self):
 return self.id[:10]
Expand Down

86 changes: 70 additions & 16 deletions
compose/service.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,14 +3,14 @@
from collections import namedtuple
import logging
import re
from operator import attrgetter
import sys
import six
from operator import attrgetter

import six
from docker.errors import APIError
from docker.utils import create_host_config, LogConfig

from .config import DOCKER_CONFIG_KEYS
from .config import DOCKER_CONFIG_KEYS, merge_environment
from .container import Container, get_container_name
from .progress_stream import stream_output, StreamOutputError

Expand DownExpand Up@@ -183,10 +183,10 @@ def create_container(self,
 Create a container for this service. If the image doesn't exist, attempt to pull
 it.
 """
 override_options['volumes_from'] = self._get_volumes_from(previous_container)
 container_options = self._get_container_create_options(
 override_options,
 one_off=one_off,
 previous_container=previous_container,
 )

 if (do_build and
Expand DownExpand Up@@ -247,6 +247,12 @@ def recreate_container(self, container, **override_options):
 self.client.rename(
 container.id,
 '%s_%s' % (container.short_id, container.name))

 override_options = dict(
 override_options,
 environment=merge_environment(
 override_options.get('environment'),
 {'affinity:container': '=' + container.id}))
 new_container = self.create_container(
 do_build=False,
 previous_container=container,
Expand DownExpand Up@@ -326,7 +332,7 @@ def _get_links(self, link_to_self):
 links.append((external_link, link_name))
 return links

 def _get_volumes_from(self, previous_container=None):
 def _get_volumes_from(self):
 volumes_from = []
 for volume_source in self.volumes_from:
 if isinstance(volume_source, Service):
Expand All@@ -339,9 +345,6 @@ def _get_volumes_from(self, previous_container=None):
 elif isinstance(volume_source, Container):
 volumes_from.append(volume_source.id)

 if previous_container:
 volumes_from.append(previous_container.id)

 return volumes_from

 def _get_net(self):
Expand All@@ -363,7 +366,11 @@ def _get_net(self):

 return net

 def _get_container_create_options(self, override_options, one_off=False):
 def _get_container_create_options(
 self,
 override_options,
 one_off=False,
 previous_container=None):
 container_options = dict(
 (k, self.options[k])
 for k in DOCKER_CONFIG_KEYS if k in self.options)
Expand DownExpand Up@@ -396,11 +403,19 @@ def _get_container_create_options(self, override_options, one_off=False):
 ports.append(port)
 container_options['ports'] = ports

 override_options['binds'] = merge_volume_bindings(
 container_options.get('volumes') or [],
 previous_container)

 if 'volumes' in container_options:
 container_options['volumes'] = dict(
 (parse_volume_spec(v).internal, {})
 for v in container_options['volumes'])

 container_options['environment'] = merge_environment(
 self.options.get('environment'),
 override_options.get('environment'))

 if self.can_be_built():
 container_options['image'] = self.full_name

Expand All@@ -418,11 +433,6 @@ def _get_container_host_config(self, override_options, one_off=False):
 options = dict(self.options, **override_options)
 port_bindings = build_port_bindings(options.get('ports') or [])

 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in options.get('volumes') or []
 if ':' in volume)

 privileged = options.get('privileged', False)
 cap_add = options.get('cap_add', None)
 cap_drop = options.get('cap_drop', None)
Expand All@@ -447,8 +457,8 @@ def _get_container_host_config(self, override_options, one_off=False):
 return create_host_config(
 links=self._get_links(link_to_self=one_off),
 port_bindings=port_bindings,
 binds=volume_bindings,
 volumes_from=options.get('volumes_from'),
 binds=options.get('binds'),
 volumes_from=self._get_volumes_from(),
 privileged=privileged,
 network_mode=self._get_net(),
 devices=devices,
Expand DownExpand Up@@ -531,6 +541,50 @@ def pull(self, insecure_registry=False):
 stream_output(output, sys.stdout)


def get_container_data_volumes(container, volumes_option):
 """Find the container data volumes that are in `volumes_option`, and return
 a mapping of volume bindings for those volumes.
 """
 volumes = []

 volumes_option = volumes_option or []
 container_volumes = container.get('Volumes') or {}
 image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {}

 for volume in set(volumes_option + image_volumes.keys()):
 volume = parse_volume_spec(volume)
 # No need to preserve host volumes
 if volume.external:
 continue

 volume_path = container_volumes.get(volume.internal)
 # New volume, doesn't exist in the old container
 if not volume_path:
 continue

 # Copy existing volume from old container
 volume = volume._replace(external=volume_path)
 volumes.append(build_volume_binding(volume))

 return dict(volumes)


def merge_volume_bindings(volumes_option, previous_container):
 """Return a list of volume bindings for a container. Container data volumes
 are replaced by those from the previous container.
 """
 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in volumes_option or []
 if ':' in volume)

 if previous_container:
 volume_bindings.update(
 get_container_data_volumes(previous_container, volumes_option))

 return volume_bindings


NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')


Expand Down

27 changes: 14 additions & 13 deletions
tests/integration/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def test_create_container_with_unspecified_volume(self):
 service = self.create_service('db', volumes=['/var/db'])
 container = service.create_container()
 service.start_container(container)
 self.assertIn('/var/db', container.inspect()['Volumes'])
 self.assertIn('/var/db', container.get('Volumes'))

 def test_create_container_with_cpu_shares(self):
 service = self.create_service('db', cpu_shares=73)
Expand DownExpand Up@@ -239,24 +239,27 @@ def test_recreate_containers(self):
 command=['300']
 )
 old_container = service.create_container()
 self.assertEqual(old_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(old_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=1', old_container.dictionary['Config']['Env'])
 self.assertEqual(old_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(old_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=1', old_container.get('Config.Env'))
 self.assertEqual(old_container.name, 'composetest_db_1')
 service.start_container(old_container)
 volume_path = old_container.inspect()['Volumes']['/etc']
 old_container.inspect() # reload volume data
 volume_path = old_container.get('Volumes')['/etc']

 num_containers_before = len(self.client.containers(all=True))

 service.options['environment']['FOO'] = '2'
 new_container, = service.recreate_containers()

 self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(new_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=2', new_container.dictionary['Config']['Env'])
 self.assertEqual(new_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(new_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=2', new_container.get('Config.Env'))
 self.assertEqual(new_container.name, 'composetest_db_1')
 self.assertEqual(new_container.inspect()['Volumes']['/etc'], volume_path)
 self.assertIn(old_container.id, new_container.dictionary['HostConfig']['VolumesFrom'])
 self.assertEqual(new_container.get('Volumes')['/etc'], volume_path)
 self.assertIn(
 'affinity:container==%s' % old_container.id,
 new_container.get('Config.Env'))

 self.assertEqual(len(self.client.containers(all=True)), num_containers_before)
 self.assertNotEqual(old_container.id, new_container.id)
Expand DownExpand Up@@ -289,9 +292,7 @@ def test_recreate_containers_with_image_declared_volume(self):
 self.assertEqual(old_container.get('Volumes').keys(), ['/data'])
 volume_path = old_container.get('Volumes')['/data']

 service.recreate_containers()
 new_container = service.containers()[0]
 service.start_container(new_container)
 new_container = service.recreate_containers()[0]
 self.assertEqual(new_container.get('Volumes').keys(), ['/data'])
 self.assertEqual(new_container.get('Volumes')['/data'], volume_path)

Expand Down

70 changes: 63 additions & 7 deletions
tests/unit/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,9 @@
 ConfigError,
 build_port_bindings,
 build_volume_binding,
 get_container_data_volumes,
 get_container_name,
 merge_volume_bindings,
 parse_repository_tag,
 parse_volume_spec,
 split_port,
Expand DownExpand Up@@ -86,13 +88,6 @@ def test_get_volumes_from_container(self):

 self.assertEqual(service._get_volumes_from(), [container_id])

 def test_get_volumes_from_previous_container(self):
 container_id = 'aabbccddee'
 service = Service('test', image='foo')
 container = mock.Mock(id=container_id, spec=Container, image='foo')

 self.assertEqual(service._get_volumes_from(container), [container_id])

 def test_get_volumes_from_service_container_exists(self):
 container_ids = ['aabbccddee', '12345']
 from_service = mock.create_autospec(Service)
Expand DownExpand Up@@ -320,6 +315,9 @@ def test_create_container_no_build(self):

class ServiceVolumesTest(unittest.TestCase):

 def setUp(self):
 self.mock_client = mock.create_autospec(docker.Client)

 def test_parse_volume_spec_only_one_path(self):
 spec = parse_volume_spec('/the/volume')
 self.assertEqual(spec, (None, '/the/volume', 'rw'))
Expand All@@ -345,3 +343,61 @@ def test_build_volume_binding(self):
 self.assertEqual(
 binding,
 ('/outside', dict(bind='/inside', ro=False)))

 def test_get_container_data_volumes(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {
 'Volumes': {
 '/mnt/image/data': {},
 }
 }
 }
 container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {
 '/host/volume': '/host/volume',
 '/existing/volume': '/var/lib/docker/aaaaaaaa',
 '/removed/volume': '/var/lib/docker/bbbbbbbb',
 '/mnt/image/data': '/var/lib/docker/cccccccc',
 },
 }, has_been_inspected=True)

 expected = {
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 '/var/lib/docker/cccccccc': {'bind': '/mnt/image/data', 'ro': False},
 }

 binds = get_container_data_volumes(container, options)
 self.assertEqual(binds, expected)

 def test_merge_volume_bindings(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/host/rw/volume:/host/rw/volume',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {'Volumes': {}}
 }

 intermediate_container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'},
 }, has_been_inspected=True)

 expected = {
 '/host/volume': {'bind': '/host/volume', 'ro': True},
 '/host/rw/volume': {'bind': '/host/rw/volume', 'ro': False},
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 }

 binds = merge_volume_bindings(options, intermediate_container)
 self.assertEqual(binds, expected)

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Preserve individual volumes on recreate by dnephin · Pull Request #858 · docker/compose · GitHub
Skip to content
Preserve individual volumes on recreate#858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

aanand
merged 1 commit into

from

May 14, 2015

+152

−40

 Merged


Preserve individual volumes on recreate
#858
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5 changes: 1 addition & 4 deletions
compose/cli/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,10 +317,7 @@ def run(self, project, options):
 }

 if options['-e']:
 # Merge environment from config with -e command line
 container_options['environment'] = dict(
 parse_environment(service.options.get('environment')),
 **parse_environment(options['-e']))
 container_options['environment'] = parse_environment(options['-e'])

 if options['--entrypoint']:
 container_options['entrypoint'] = options.get('--entrypoint')
Expand Down

4 changes: 4 additions & 0 deletions
compose/container.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -44,6 +44,10 @@ def id(self):
 def image(self):
 return self.dictionary['Image']

 @property
 def image_config(self):
 return self.client.inspect_image(self.image)

 @property
 def short_id(self):
 return self.id[:10]
Expand Down

86 changes: 70 additions & 16 deletions
compose/service.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,14 +3,14 @@
from collections import namedtuple
import logging
import re
from operator import attrgetter
import sys
import six
from operator import attrgetter

import six
from docker.errors import APIError
from docker.utils import create_host_config, LogConfig

from .config import DOCKER_CONFIG_KEYS
from .config import DOCKER_CONFIG_KEYS, merge_environment
from .container import Container, get_container_name
from .progress_stream import stream_output, StreamOutputError

Expand DownExpand Up@@ -183,10 +183,10 @@ def create_container(self,
 Create a container for this service. If the image doesn't exist, attempt to pull
 it.
 """
 override_options['volumes_from'] = self._get_volumes_from(previous_container)
 container_options = self._get_container_create_options(
 override_options,
 one_off=one_off,
 previous_container=previous_container,
 )

 if (do_build and
Expand DownExpand Up@@ -247,6 +247,12 @@ def recreate_container(self, container, **override_options):
 self.client.rename(
 container.id,
 '%s_%s' % (container.short_id, container.name))

 override_options = dict(
 override_options,
 environment=merge_environment(
 override_options.get('environment'),
 {'affinity:container': '=' + container.id}))
 new_container = self.create_container(
 do_build=False,
 previous_container=container,
Expand DownExpand Up@@ -326,7 +332,7 @@ def _get_links(self, link_to_self):
 links.append((external_link, link_name))
 return links

 def _get_volumes_from(self, previous_container=None):
 def _get_volumes_from(self):
 volumes_from = []
 for volume_source in self.volumes_from:
 if isinstance(volume_source, Service):
Expand All@@ -339,9 +345,6 @@ def _get_volumes_from(self, previous_container=None):
 elif isinstance(volume_source, Container):
 volumes_from.append(volume_source.id)

 if previous_container:
 volumes_from.append(previous_container.id)

 return volumes_from

 def _get_net(self):
Expand All@@ -363,7 +366,11 @@ def _get_net(self):

 return net

 def _get_container_create_options(self, override_options, one_off=False):
 def _get_container_create_options(
 self,
 override_options,
 one_off=False,
 previous_container=None):
 container_options = dict(
 (k, self.options[k])
 for k in DOCKER_CONFIG_KEYS if k in self.options)
Expand DownExpand Up@@ -396,11 +403,19 @@ def _get_container_create_options(self, override_options, one_off=False):
 ports.append(port)
 container_options['ports'] = ports

 override_options['binds'] = merge_volume_bindings(
 container_options.get('volumes') or [],
 previous_container)

 if 'volumes' in container_options:
 container_options['volumes'] = dict(
 (parse_volume_spec(v).internal, {})
 for v in container_options['volumes'])

 container_options['environment'] = merge_environment(
 self.options.get('environment'),
 override_options.get('environment'))

 if self.can_be_built():
 container_options['image'] = self.full_name

Expand All@@ -418,11 +433,6 @@ def _get_container_host_config(self, override_options, one_off=False):
 options = dict(self.options, **override_options)
 port_bindings = build_port_bindings(options.get('ports') or [])

 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in options.get('volumes') or []
 if ':' in volume)

 privileged = options.get('privileged', False)
 cap_add = options.get('cap_add', None)
 cap_drop = options.get('cap_drop', None)
Expand All@@ -447,8 +457,8 @@ def _get_container_host_config(self, override_options, one_off=False):
 return create_host_config(
 links=self._get_links(link_to_self=one_off),
 port_bindings=port_bindings,
 binds=volume_bindings,
 volumes_from=options.get('volumes_from'),
 binds=options.get('binds'),
 volumes_from=self._get_volumes_from(),
 privileged=privileged,
 network_mode=self._get_net(),
 devices=devices,
Expand DownExpand Up@@ -531,6 +541,50 @@ def pull(self, insecure_registry=False):
 stream_output(output, sys.stdout)


def get_container_data_volumes(container, volumes_option):
 """Find the container data volumes that are in `volumes_option`, and return
 a mapping of volume bindings for those volumes.
 """
 volumes = []

 volumes_option = volumes_option or []
 container_volumes = container.get('Volumes') or {}
 image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {}

 for volume in set(volumes_option + image_volumes.keys()):
 volume = parse_volume_spec(volume)
 # No need to preserve host volumes
 if volume.external:
 continue

 volume_path = container_volumes.get(volume.internal)
 # New volume, doesn't exist in the old container
 if not volume_path:
 continue

 # Copy existing volume from old container
 volume = volume._replace(external=volume_path)
 volumes.append(build_volume_binding(volume))

 return dict(volumes)


def merge_volume_bindings(volumes_option, previous_container):
 """Return a list of volume bindings for a container. Container data volumes
 are replaced by those from the previous container.
 """
 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in volumes_option or []
 if ':' in volume)

 if previous_container:
 volume_bindings.update(
 get_container_data_volumes(previous_container, volumes_option))

 return volume_bindings


NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')


Expand Down

27 changes: 14 additions & 13 deletions
tests/integration/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def test_create_container_with_unspecified_volume(self):
 service = self.create_service('db', volumes=['/var/db'])
 container = service.create_container()
 service.start_container(container)
 self.assertIn('/var/db', container.inspect()['Volumes'])
 self.assertIn('/var/db', container.get('Volumes'))

 def test_create_container_with_cpu_shares(self):
 service = self.create_service('db', cpu_shares=73)
Expand DownExpand Up@@ -239,24 +239,27 @@ def test_recreate_containers(self):
 command=['300']
 )
 old_container = service.create_container()
 self.assertEqual(old_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(old_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=1', old_container.dictionary['Config']['Env'])
 self.assertEqual(old_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(old_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=1', old_container.get('Config.Env'))
 self.assertEqual(old_container.name, 'composetest_db_1')
 service.start_container(old_container)
 volume_path = old_container.inspect()['Volumes']['/etc']
 old_container.inspect() # reload volume data
 volume_path = old_container.get('Volumes')['/etc']

 num_containers_before = len(self.client.containers(all=True))

 service.options['environment']['FOO'] = '2'
 new_container, = service.recreate_containers()

 self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(new_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=2', new_container.dictionary['Config']['Env'])
 self.assertEqual(new_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(new_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=2', new_container.get('Config.Env'))
 self.assertEqual(new_container.name, 'composetest_db_1')
 self.assertEqual(new_container.inspect()['Volumes']['/etc'], volume_path)
 self.assertIn(old_container.id, new_container.dictionary['HostConfig']['VolumesFrom'])
 self.assertEqual(new_container.get('Volumes')['/etc'], volume_path)
 self.assertIn(
 'affinity:container==%s' % old_container.id,
 new_container.get('Config.Env'))

 self.assertEqual(len(self.client.containers(all=True)), num_containers_before)
 self.assertNotEqual(old_container.id, new_container.id)
Expand DownExpand Up@@ -289,9 +292,7 @@ def test_recreate_containers_with_image_declared_volume(self):
 self.assertEqual(old_container.get('Volumes').keys(), ['/data'])
 volume_path = old_container.get('Volumes')['/data']

 service.recreate_containers()
 new_container = service.containers()[0]
 service.start_container(new_container)
 new_container = service.recreate_containers()[0]
 self.assertEqual(new_container.get('Volumes').keys(), ['/data'])
 self.assertEqual(new_container.get('Volumes')['/data'], volume_path)

Expand Down

70 changes: 63 additions & 7 deletions
tests/unit/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,9 @@
 ConfigError,
 build_port_bindings,
 build_volume_binding,
 get_container_data_volumes,
 get_container_name,
 merge_volume_bindings,
 parse_repository_tag,
 parse_volume_spec,
 split_port,
Expand DownExpand Up@@ -86,13 +88,6 @@ def test_get_volumes_from_container(self):

 self.assertEqual(service._get_volumes_from(), [container_id])

 def test_get_volumes_from_previous_container(self):
 container_id = 'aabbccddee'
 service = Service('test', image='foo')
 container = mock.Mock(id=container_id, spec=Container, image='foo')

 self.assertEqual(service._get_volumes_from(container), [container_id])

 def test_get_volumes_from_service_container_exists(self):
 container_ids = ['aabbccddee', '12345']
 from_service = mock.create_autospec(Service)
Expand DownExpand Up@@ -320,6 +315,9 @@ def test_create_container_no_build(self):

class ServiceVolumesTest(unittest.TestCase):

 def setUp(self):
 self.mock_client = mock.create_autospec(docker.Client)

 def test_parse_volume_spec_only_one_path(self):
 spec = parse_volume_spec('/the/volume')
 self.assertEqual(spec, (None, '/the/volume', 'rw'))
Expand All@@ -345,3 +343,61 @@ def test_build_volume_binding(self):
 self.assertEqual(
 binding,
 ('/outside', dict(bind='/inside', ro=False)))

 def test_get_container_data_volumes(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {
 'Volumes': {
 '/mnt/image/data': {},
 }
 }
 }
 container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {
 '/host/volume': '/host/volume',
 '/existing/volume': '/var/lib/docker/aaaaaaaa',
 '/removed/volume': '/var/lib/docker/bbbbbbbb',
 '/mnt/image/data': '/var/lib/docker/cccccccc',
 },
 }, has_been_inspected=True)

 expected = {
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 '/var/lib/docker/cccccccc': {'bind': '/mnt/image/data', 'ro': False},
 }

 binds = get_container_data_volumes(container, options)
 self.assertEqual(binds, expected)

 def test_merge_volume_bindings(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/host/rw/volume:/host/rw/volume',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {'Volumes': {}}
 }

 intermediate_container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'},
 }, has_been_inspected=True)

 expected = {
 '/host/volume': {'bind': '/host/volume', 'ro': True},
 '/host/rw/volume': {'bind': '/host/rw/volume', 'ro': False},
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 }

 binds = merge_volume_bindings(options, intermediate_container)
 self.assertEqual(binds, expected)

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Preserve individual volumes on recreate by dnephin · Pull Request #858 · docker/compose · GitHub
Skip to content
Preserve individual volumes on recreate#858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

aanand
merged 1 commit into

from

May 14, 2015

+152

−40

 Merged


Preserve individual volumes on recreate
#858
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5 changes: 1 addition & 4 deletions
compose/cli/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,10 +317,7 @@ def run(self, project, options):
 }

 if options['-e']:
 # Merge environment from config with -e command line
 container_options['environment'] = dict(
 parse_environment(service.options.get('environment')),
 **parse_environment(options['-e']))
 container_options['environment'] = parse_environment(options['-e'])

 if options['--entrypoint']:
 container_options['entrypoint'] = options.get('--entrypoint')
Expand Down

4 changes: 4 additions & 0 deletions
compose/container.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -44,6 +44,10 @@ def id(self):
 def image(self):
 return self.dictionary['Image']

 @property
 def image_config(self):
 return self.client.inspect_image(self.image)

 @property
 def short_id(self):
 return self.id[:10]
Expand Down

86 changes: 70 additions & 16 deletions
compose/service.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,14 +3,14 @@
from collections import namedtuple
import logging
import re
from operator import attrgetter
import sys
import six
from operator import attrgetter

import six
from docker.errors import APIError
from docker.utils import create_host_config, LogConfig

from .config import DOCKER_CONFIG_KEYS
from .config import DOCKER_CONFIG_KEYS, merge_environment
from .container import Container, get_container_name
from .progress_stream import stream_output, StreamOutputError

Expand DownExpand Up@@ -183,10 +183,10 @@ def create_container(self,
 Create a container for this service. If the image doesn't exist, attempt to pull
 it.
 """
 override_options['volumes_from'] = self._get_volumes_from(previous_container)
 container_options = self._get_container_create_options(
 override_options,
 one_off=one_off,
 previous_container=previous_container,
 )

 if (do_build and
Expand DownExpand Up@@ -247,6 +247,12 @@ def recreate_container(self, container, **override_options):
 self.client.rename(
 container.id,
 '%s_%s' % (container.short_id, container.name))

 override_options = dict(
 override_options,
 environment=merge_environment(
 override_options.get('environment'),
 {'affinity:container': '=' + container.id}))
 new_container = self.create_container(
 do_build=False,
 previous_container=container,
Expand DownExpand Up@@ -326,7 +332,7 @@ def _get_links(self, link_to_self):
 links.append((external_link, link_name))
 return links

 def _get_volumes_from(self, previous_container=None):
 def _get_volumes_from(self):
 volumes_from = []
 for volume_source in self.volumes_from:
 if isinstance(volume_source, Service):
Expand All@@ -339,9 +345,6 @@ def _get_volumes_from(self, previous_container=None):
 elif isinstance(volume_source, Container):
 volumes_from.append(volume_source.id)

 if previous_container:
 volumes_from.append(previous_container.id)

 return volumes_from

 def _get_net(self):
Expand All@@ -363,7 +366,11 @@ def _get_net(self):

 return net

 def _get_container_create_options(self, override_options, one_off=False):
 def _get_container_create_options(
 self,
 override_options,
 one_off=False,
 previous_container=None):
 container_options = dict(
 (k, self.options[k])
 for k in DOCKER_CONFIG_KEYS if k in self.options)
Expand DownExpand Up@@ -396,11 +403,19 @@ def _get_container_create_options(self, override_options, one_off=False):
 ports.append(port)
 container_options['ports'] = ports

 override_options['binds'] = merge_volume_bindings(
 container_options.get('volumes') or [],
 previous_container)

 if 'volumes' in container_options:
 container_options['volumes'] = dict(
 (parse_volume_spec(v).internal, {})
 for v in container_options['volumes'])

 container_options['environment'] = merge_environment(
 self.options.get('environment'),
 override_options.get('environment'))

 if self.can_be_built():
 container_options['image'] = self.full_name

Expand All@@ -418,11 +433,6 @@ def _get_container_host_config(self, override_options, one_off=False):
 options = dict(self.options, **override_options)
 port_bindings = build_port_bindings(options.get('ports') or [])

 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in options.get('volumes') or []
 if ':' in volume)

 privileged = options.get('privileged', False)
 cap_add = options.get('cap_add', None)
 cap_drop = options.get('cap_drop', None)
Expand All@@ -447,8 +457,8 @@ def _get_container_host_config(self, override_options, one_off=False):
 return create_host_config(
 links=self._get_links(link_to_self=one_off),
 port_bindings=port_bindings,
 binds=volume_bindings,
 volumes_from=options.get('volumes_from'),
 binds=options.get('binds'),
 volumes_from=self._get_volumes_from(),
 privileged=privileged,
 network_mode=self._get_net(),
 devices=devices,
Expand DownExpand Up@@ -531,6 +541,50 @@ def pull(self, insecure_registry=False):
 stream_output(output, sys.stdout)


def get_container_data_volumes(container, volumes_option):
 """Find the container data volumes that are in `volumes_option`, and return
 a mapping of volume bindings for those volumes.
 """
 volumes = []

 volumes_option = volumes_option or []
 container_volumes = container.get('Volumes') or {}
 image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {}

 for volume in set(volumes_option + image_volumes.keys()):
 volume = parse_volume_spec(volume)
 # No need to preserve host volumes
 if volume.external:
 continue

 volume_path = container_volumes.get(volume.internal)
 # New volume, doesn't exist in the old container
 if not volume_path:
 continue

 # Copy existing volume from old container
 volume = volume._replace(external=volume_path)
 volumes.append(build_volume_binding(volume))

 return dict(volumes)


def merge_volume_bindings(volumes_option, previous_container):
 """Return a list of volume bindings for a container. Container data volumes
 are replaced by those from the previous container.
 """
 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in volumes_option or []
 if ':' in volume)

 if previous_container:
 volume_bindings.update(
 get_container_data_volumes(previous_container, volumes_option))

 return volume_bindings


NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')


Expand Down

27 changes: 14 additions & 13 deletions
tests/integration/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def test_create_container_with_unspecified_volume(self):
 service = self.create_service('db', volumes=['/var/db'])
 container = service.create_container()
 service.start_container(container)
 self.assertIn('/var/db', container.inspect()['Volumes'])
 self.assertIn('/var/db', container.get('Volumes'))

 def test_create_container_with_cpu_shares(self):
 service = self.create_service('db', cpu_shares=73)
Expand DownExpand Up@@ -239,24 +239,27 @@ def test_recreate_containers(self):
 command=['300']
 )
 old_container = service.create_container()
 self.assertEqual(old_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(old_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=1', old_container.dictionary['Config']['Env'])
 self.assertEqual(old_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(old_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=1', old_container.get('Config.Env'))
 self.assertEqual(old_container.name, 'composetest_db_1')
 service.start_container(old_container)
 volume_path = old_container.inspect()['Volumes']['/etc']
 old_container.inspect() # reload volume data
 volume_path = old_container.get('Volumes')['/etc']

 num_containers_before = len(self.client.containers(all=True))

 service.options['environment']['FOO'] = '2'
 new_container, = service.recreate_containers()

 self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(new_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=2', new_container.dictionary['Config']['Env'])
 self.assertEqual(new_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(new_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=2', new_container.get('Config.Env'))
 self.assertEqual(new_container.name, 'composetest_db_1')
 self.assertEqual(new_container.inspect()['Volumes']['/etc'], volume_path)
 self.assertIn(old_container.id, new_container.dictionary['HostConfig']['VolumesFrom'])
 self.assertEqual(new_container.get('Volumes')['/etc'], volume_path)
 self.assertIn(
 'affinity:container==%s' % old_container.id,
 new_container.get('Config.Env'))

 self.assertEqual(len(self.client.containers(all=True)), num_containers_before)
 self.assertNotEqual(old_container.id, new_container.id)
Expand DownExpand Up@@ -289,9 +292,7 @@ def test_recreate_containers_with_image_declared_volume(self):
 self.assertEqual(old_container.get('Volumes').keys(), ['/data'])
 volume_path = old_container.get('Volumes')['/data']

 service.recreate_containers()
 new_container = service.containers()[0]
 service.start_container(new_container)
 new_container = service.recreate_containers()[0]
 self.assertEqual(new_container.get('Volumes').keys(), ['/data'])
 self.assertEqual(new_container.get('Volumes')['/data'], volume_path)

Expand Down

70 changes: 63 additions & 7 deletions
tests/unit/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,9 @@
 ConfigError,
 build_port_bindings,
 build_volume_binding,
 get_container_data_volumes,
 get_container_name,
 merge_volume_bindings,
 parse_repository_tag,
 parse_volume_spec,
 split_port,
Expand DownExpand Up@@ -86,13 +88,6 @@ def test_get_volumes_from_container(self):

 self.assertEqual(service._get_volumes_from(), [container_id])

 def test_get_volumes_from_previous_container(self):
 container_id = 'aabbccddee'
 service = Service('test', image='foo')
 container = mock.Mock(id=container_id, spec=Container, image='foo')

 self.assertEqual(service._get_volumes_from(container), [container_id])

 def test_get_volumes_from_service_container_exists(self):
 container_ids = ['aabbccddee', '12345']
 from_service = mock.create_autospec(Service)
Expand DownExpand Up@@ -320,6 +315,9 @@ def test_create_container_no_build(self):

class ServiceVolumesTest(unittest.TestCase):

 def setUp(self):
 self.mock_client = mock.create_autospec(docker.Client)

 def test_parse_volume_spec_only_one_path(self):
 spec = parse_volume_spec('/the/volume')
 self.assertEqual(spec, (None, '/the/volume', 'rw'))
Expand All@@ -345,3 +343,61 @@ def test_build_volume_binding(self):
 self.assertEqual(
 binding,
 ('/outside', dict(bind='/inside', ro=False)))

 def test_get_container_data_volumes(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {
 'Volumes': {
 '/mnt/image/data': {},
 }
 }
 }
 container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {
 '/host/volume': '/host/volume',
 '/existing/volume': '/var/lib/docker/aaaaaaaa',
 '/removed/volume': '/var/lib/docker/bbbbbbbb',
 '/mnt/image/data': '/var/lib/docker/cccccccc',
 },
 }, has_been_inspected=True)

 expected = {
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 '/var/lib/docker/cccccccc': {'bind': '/mnt/image/data', 'ro': False},
 }

 binds = get_container_data_volumes(container, options)
 self.assertEqual(binds, expected)

 def test_merge_volume_bindings(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/host/rw/volume:/host/rw/volume',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {'Volumes': {}}
 }

 intermediate_container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'},
 }, has_been_inspected=True)

 expected = {
 '/host/volume': {'bind': '/host/volume', 'ro': True},
 '/host/rw/volume': {'bind': '/host/rw/volume', 'ro': False},
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 }

 binds = merge_volume_bindings(options, intermediate_container)
 self.assertEqual(binds, expected)

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
Preserve individual volumes on recreate by dnephin · Pull Request #858 · docker/compose · GitHub
Skip to content
Preserve individual volumes on recreate#858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

aanand
merged 1 commit into

from

May 14, 2015

+152

−40

 Merged


Preserve individual volumes on recreate
#858
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5 changes: 1 addition & 4 deletions
compose/cli/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,10 +317,7 @@ def run(self, project, options):
 }

 if options['-e']:
 # Merge environment from config with -e command line
 container_options['environment'] = dict(
 parse_environment(service.options.get('environment')),
 **parse_environment(options['-e']))
 container_options['environment'] = parse_environment(options['-e'])

 if options['--entrypoint']:
 container_options['entrypoint'] = options.get('--entrypoint')
Expand Down

4 changes: 4 additions & 0 deletions
compose/container.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -44,6 +44,10 @@ def id(self):
 def image(self):
 return self.dictionary['Image']

 @property
 def image_config(self):
 return self.client.inspect_image(self.image)

 @property
 def short_id(self):
 return self.id[:10]
Expand Down

86 changes: 70 additions & 16 deletions
compose/service.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,14 +3,14 @@
from collections import namedtuple
import logging
import re
from operator import attrgetter
import sys
import six
from operator import attrgetter

import six
from docker.errors import APIError
from docker.utils import create_host_config, LogConfig

from .config import DOCKER_CONFIG_KEYS
from .config import DOCKER_CONFIG_KEYS, merge_environment
from .container import Container, get_container_name
from .progress_stream import stream_output, StreamOutputError

Expand DownExpand Up@@ -183,10 +183,10 @@ def create_container(self,
 Create a container for this service. If the image doesn't exist, attempt to pull
 it.
 """
 override_options['volumes_from'] = self._get_volumes_from(previous_container)
 container_options = self._get_container_create_options(
 override_options,
 one_off=one_off,
 previous_container=previous_container,
 )

 if (do_build and
Expand DownExpand Up@@ -247,6 +247,12 @@ def recreate_container(self, container, **override_options):
 self.client.rename(
 container.id,
 '%s_%s' % (container.short_id, container.name))

 override_options = dict(
 override_options,
 environment=merge_environment(
 override_options.get('environment'),
 {'affinity:container': '=' + container.id}))
 new_container = self.create_container(
 do_build=False,
 previous_container=container,
Expand DownExpand Up@@ -326,7 +332,7 @@ def _get_links(self, link_to_self):
 links.append((external_link, link_name))
 return links

 def _get_volumes_from(self, previous_container=None):
 def _get_volumes_from(self):
 volumes_from = []
 for volume_source in self.volumes_from:
 if isinstance(volume_source, Service):
Expand All@@ -339,9 +345,6 @@ def _get_volumes_from(self, previous_container=None):
 elif isinstance(volume_source, Container):
 volumes_from.append(volume_source.id)

 if previous_container:
 volumes_from.append(previous_container.id)

 return volumes_from

 def _get_net(self):
Expand All@@ -363,7 +366,11 @@ def _get_net(self):

 return net

 def _get_container_create_options(self, override_options, one_off=False):
 def _get_container_create_options(
 self,
 override_options,
 one_off=False,
 previous_container=None):
 container_options = dict(
 (k, self.options[k])
 for k in DOCKER_CONFIG_KEYS if k in self.options)
Expand DownExpand Up@@ -396,11 +403,19 @@ def _get_container_create_options(self, override_options, one_off=False):
 ports.append(port)
 container_options['ports'] = ports

 override_options['binds'] = merge_volume_bindings(
 container_options.get('volumes') or [],
 previous_container)

 if 'volumes' in container_options:
 container_options['volumes'] = dict(
 (parse_volume_spec(v).internal, {})
 for v in container_options['volumes'])

 container_options['environment'] = merge_environment(
 self.options.get('environment'),
 override_options.get('environment'))

 if self.can_be_built():
 container_options['image'] = self.full_name

Expand All@@ -418,11 +433,6 @@ def _get_container_host_config(self, override_options, one_off=False):
 options = dict(self.options, **override_options)
 port_bindings = build_port_bindings(options.get('ports') or [])

 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in options.get('volumes') or []
 if ':' in volume)

 privileged = options.get('privileged', False)
 cap_add = options.get('cap_add', None)
 cap_drop = options.get('cap_drop', None)
Expand All@@ -447,8 +457,8 @@ def _get_container_host_config(self, override_options, one_off=False):
 return create_host_config(
 links=self._get_links(link_to_self=one_off),
 port_bindings=port_bindings,
 binds=volume_bindings,
 volumes_from=options.get('volumes_from'),
 binds=options.get('binds'),
 volumes_from=self._get_volumes_from(),
 privileged=privileged,
 network_mode=self._get_net(),
 devices=devices,
Expand DownExpand Up@@ -531,6 +541,50 @@ def pull(self, insecure_registry=False):
 stream_output(output, sys.stdout)


def get_container_data_volumes(container, volumes_option):
 """Find the container data volumes that are in `volumes_option`, and return
 a mapping of volume bindings for those volumes.
 """
 volumes = []

 volumes_option = volumes_option or []
 container_volumes = container.get('Volumes') or {}
 image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {}

 for volume in set(volumes_option + image_volumes.keys()):
 volume = parse_volume_spec(volume)
 # No need to preserve host volumes
 if volume.external:
 continue

 volume_path = container_volumes.get(volume.internal)
 # New volume, doesn't exist in the old container
 if not volume_path:
 continue

 # Copy existing volume from old container
 volume = volume._replace(external=volume_path)
 volumes.append(build_volume_binding(volume))

 return dict(volumes)


def merge_volume_bindings(volumes_option, previous_container):
 """Return a list of volume bindings for a container. Container data volumes
 are replaced by those from the previous container.
 """
 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in volumes_option or []
 if ':' in volume)

 if previous_container:
 volume_bindings.update(
 get_container_data_volumes(previous_container, volumes_option))

 return volume_bindings


NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')


Expand Down

27 changes: 14 additions & 13 deletions
tests/integration/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def test_create_container_with_unspecified_volume(self):
 service = self.create_service('db', volumes=['/var/db'])
 container = service.create_container()
 service.start_container(container)
 self.assertIn('/var/db', container.inspect()['Volumes'])
 self.assertIn('/var/db', container.get('Volumes'))

 def test_create_container_with_cpu_shares(self):
 service = self.create_service('db', cpu_shares=73)
Expand DownExpand Up@@ -239,24 +239,27 @@ def test_recreate_containers(self):
 command=['300']
 )
 old_container = service.create_container()
 self.assertEqual(old_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(old_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=1', old_container.dictionary['Config']['Env'])
 self.assertEqual(old_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(old_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=1', old_container.get('Config.Env'))
 self.assertEqual(old_container.name, 'composetest_db_1')
 service.start_container(old_container)
 volume_path = old_container.inspect()['Volumes']['/etc']
 old_container.inspect() # reload volume data
 volume_path = old_container.get('Volumes')['/etc']

 num_containers_before = len(self.client.containers(all=True))

 service.options['environment']['FOO'] = '2'
 new_container, = service.recreate_containers()

 self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(new_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=2', new_container.dictionary['Config']['Env'])
 self.assertEqual(new_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(new_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=2', new_container.get('Config.Env'))
 self.assertEqual(new_container.name, 'composetest_db_1')
 self.assertEqual(new_container.inspect()['Volumes']['/etc'], volume_path)
 self.assertIn(old_container.id, new_container.dictionary['HostConfig']['VolumesFrom'])
 self.assertEqual(new_container.get('Volumes')['/etc'], volume_path)
 self.assertIn(
 'affinity:container==%s' % old_container.id,
 new_container.get('Config.Env'))

 self.assertEqual(len(self.client.containers(all=True)), num_containers_before)
 self.assertNotEqual(old_container.id, new_container.id)
Expand DownExpand Up@@ -289,9 +292,7 @@ def test_recreate_containers_with_image_declared_volume(self):
 self.assertEqual(old_container.get('Volumes').keys(), ['/data'])
 volume_path = old_container.get('Volumes')['/data']

 service.recreate_containers()
 new_container = service.containers()[0]
 service.start_container(new_container)
 new_container = service.recreate_containers()[0]
 self.assertEqual(new_container.get('Volumes').keys(), ['/data'])
 self.assertEqual(new_container.get('Volumes')['/data'], volume_path)

Expand Down

70 changes: 63 additions & 7 deletions
tests/unit/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,9 @@
 ConfigError,
 build_port_bindings,
 build_volume_binding,
 get_container_data_volumes,
 get_container_name,
 merge_volume_bindings,
 parse_repository_tag,
 parse_volume_spec,
 split_port,
Expand DownExpand Up@@ -86,13 +88,6 @@ def test_get_volumes_from_container(self):

 self.assertEqual(service._get_volumes_from(), [container_id])

 def test_get_volumes_from_previous_container(self):
 container_id = 'aabbccddee'
 service = Service('test', image='foo')
 container = mock.Mock(id=container_id, spec=Container, image='foo')

 self.assertEqual(service._get_volumes_from(container), [container_id])

 def test_get_volumes_from_service_container_exists(self):
 container_ids = ['aabbccddee', '12345']
 from_service = mock.create_autospec(Service)
Expand DownExpand Up@@ -320,6 +315,9 @@ def test_create_container_no_build(self):

class ServiceVolumesTest(unittest.TestCase):

 def setUp(self):
 self.mock_client = mock.create_autospec(docker.Client)

 def test_parse_volume_spec_only_one_path(self):
 spec = parse_volume_spec('/the/volume')
 self.assertEqual(spec, (None, '/the/volume', 'rw'))
Expand All@@ -345,3 +343,61 @@ def test_build_volume_binding(self):
 self.assertEqual(
 binding,
 ('/outside', dict(bind='/inside', ro=False)))

 def test_get_container_data_volumes(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {
 'Volumes': {
 '/mnt/image/data': {},
 }
 }
 }
 container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {
 '/host/volume': '/host/volume',
 '/existing/volume': '/var/lib/docker/aaaaaaaa',
 '/removed/volume': '/var/lib/docker/bbbbbbbb',
 '/mnt/image/data': '/var/lib/docker/cccccccc',
 },
 }, has_been_inspected=True)

 expected = {
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 '/var/lib/docker/cccccccc': {'bind': '/mnt/image/data', 'ro': False},
 }

 binds = get_container_data_volumes(container, options)
 self.assertEqual(binds, expected)

 def test_merge_volume_bindings(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/host/rw/volume:/host/rw/volume',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {'Volumes': {}}
 }

 intermediate_container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'},
 }, has_been_inspected=True)

 expected = {
 '/host/volume': {'bind': '/host/volume', 'ro': True},
 '/host/rw/volume': {'bind': '/host/rw/volume', 'ro': False},
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 }

 binds = merge_volume_bindings(options, intermediate_container)
 self.assertEqual(binds, expected)

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Preserve individual volumes on recreate by dnephin · Pull Request #858 · docker/compose · GitHub
Skip to content
Preserve individual volumes on recreate#858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

aanand
merged 1 commit into

from

May 14, 2015

+152

−40

 Merged


Preserve individual volumes on recreate
#858
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5 changes: 1 addition & 4 deletions
compose/cli/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,10 +317,7 @@ def run(self, project, options):
 }

 if options['-e']:
 # Merge environment from config with -e command line
 container_options['environment'] = dict(
 parse_environment(service.options.get('environment')),
 **parse_environment(options['-e']))
 container_options['environment'] = parse_environment(options['-e'])

 if options['--entrypoint']:
 container_options['entrypoint'] = options.get('--entrypoint')
Expand Down

4 changes: 4 additions & 0 deletions
compose/container.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -44,6 +44,10 @@ def id(self):
 def image(self):
 return self.dictionary['Image']

 @property
 def image_config(self):
 return self.client.inspect_image(self.image)

 @property
 def short_id(self):
 return self.id[:10]
Expand Down

86 changes: 70 additions & 16 deletions
compose/service.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,14 +3,14 @@
from collections import namedtuple
import logging
import re
from operator import attrgetter
import sys
import six
from operator import attrgetter

import six
from docker.errors import APIError
from docker.utils import create_host_config, LogConfig

from .config import DOCKER_CONFIG_KEYS
from .config import DOCKER_CONFIG_KEYS, merge_environment
from .container import Container, get_container_name
from .progress_stream import stream_output, StreamOutputError

Expand DownExpand Up@@ -183,10 +183,10 @@ def create_container(self,
 Create a container for this service. If the image doesn't exist, attempt to pull
 it.
 """
 override_options['volumes_from'] = self._get_volumes_from(previous_container)
 container_options = self._get_container_create_options(
 override_options,
 one_off=one_off,
 previous_container=previous_container,
 )

 if (do_build and
Expand DownExpand Up@@ -247,6 +247,12 @@ def recreate_container(self, container, **override_options):
 self.client.rename(
 container.id,
 '%s_%s' % (container.short_id, container.name))

 override_options = dict(
 override_options,
 environment=merge_environment(
 override_options.get('environment'),
 {'affinity:container': '=' + container.id}))
 new_container = self.create_container(
 do_build=False,
 previous_container=container,
Expand DownExpand Up@@ -326,7 +332,7 @@ def _get_links(self, link_to_self):
 links.append((external_link, link_name))
 return links

 def _get_volumes_from(self, previous_container=None):
 def _get_volumes_from(self):
 volumes_from = []
 for volume_source in self.volumes_from:
 if isinstance(volume_source, Service):
Expand All@@ -339,9 +345,6 @@ def _get_volumes_from(self, previous_container=None):
 elif isinstance(volume_source, Container):
 volumes_from.append(volume_source.id)

 if previous_container:
 volumes_from.append(previous_container.id)

 return volumes_from

 def _get_net(self):
Expand All@@ -363,7 +366,11 @@ def _get_net(self):

 return net

 def _get_container_create_options(self, override_options, one_off=False):
 def _get_container_create_options(
 self,
 override_options,
 one_off=False,
 previous_container=None):
 container_options = dict(
 (k, self.options[k])
 for k in DOCKER_CONFIG_KEYS if k in self.options)
Expand DownExpand Up@@ -396,11 +403,19 @@ def _get_container_create_options(self, override_options, one_off=False):
 ports.append(port)
 container_options['ports'] = ports

 override_options['binds'] = merge_volume_bindings(
 container_options.get('volumes') or [],
 previous_container)

 if 'volumes' in container_options:
 container_options['volumes'] = dict(
 (parse_volume_spec(v).internal, {})
 for v in container_options['volumes'])

 container_options['environment'] = merge_environment(
 self.options.get('environment'),
 override_options.get('environment'))

 if self.can_be_built():
 container_options['image'] = self.full_name

Expand All@@ -418,11 +433,6 @@ def _get_container_host_config(self, override_options, one_off=False):
 options = dict(self.options, **override_options)
 port_bindings = build_port_bindings(options.get('ports') or [])

 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in options.get('volumes') or []
 if ':' in volume)

 privileged = options.get('privileged', False)
 cap_add = options.get('cap_add', None)
 cap_drop = options.get('cap_drop', None)
Expand All@@ -447,8 +457,8 @@ def _get_container_host_config(self, override_options, one_off=False):
 return create_host_config(
 links=self._get_links(link_to_self=one_off),
 port_bindings=port_bindings,
 binds=volume_bindings,
 volumes_from=options.get('volumes_from'),
 binds=options.get('binds'),
 volumes_from=self._get_volumes_from(),
 privileged=privileged,
 network_mode=self._get_net(),
 devices=devices,
Expand DownExpand Up@@ -531,6 +541,50 @@ def pull(self, insecure_registry=False):
 stream_output(output, sys.stdout)


def get_container_data_volumes(container, volumes_option):
 """Find the container data volumes that are in `volumes_option`, and return
 a mapping of volume bindings for those volumes.
 """
 volumes = []

 volumes_option = volumes_option or []
 container_volumes = container.get('Volumes') or {}
 image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {}

 for volume in set(volumes_option + image_volumes.keys()):
 volume = parse_volume_spec(volume)
 # No need to preserve host volumes
 if volume.external:
 continue

 volume_path = container_volumes.get(volume.internal)
 # New volume, doesn't exist in the old container
 if not volume_path:
 continue

 # Copy existing volume from old container
 volume = volume._replace(external=volume_path)
 volumes.append(build_volume_binding(volume))

 return dict(volumes)


def merge_volume_bindings(volumes_option, previous_container):
 """Return a list of volume bindings for a container. Container data volumes
 are replaced by those from the previous container.
 """
 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in volumes_option or []
 if ':' in volume)

 if previous_container:
 volume_bindings.update(
 get_container_data_volumes(previous_container, volumes_option))

 return volume_bindings


NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')


Expand Down

27 changes: 14 additions & 13 deletions
tests/integration/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def test_create_container_with_unspecified_volume(self):
 service = self.create_service('db', volumes=['/var/db'])
 container = service.create_container()
 service.start_container(container)
 self.assertIn('/var/db', container.inspect()['Volumes'])
 self.assertIn('/var/db', container.get('Volumes'))

 def test_create_container_with_cpu_shares(self):
 service = self.create_service('db', cpu_shares=73)
Expand DownExpand Up@@ -239,24 +239,27 @@ def test_recreate_containers(self):
 command=['300']
 )
 old_container = service.create_container()
 self.assertEqual(old_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(old_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=1', old_container.dictionary['Config']['Env'])
 self.assertEqual(old_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(old_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=1', old_container.get('Config.Env'))
 self.assertEqual(old_container.name, 'composetest_db_1')
 service.start_container(old_container)
 volume_path = old_container.inspect()['Volumes']['/etc']
 old_container.inspect() # reload volume data
 volume_path = old_container.get('Volumes')['/etc']

 num_containers_before = len(self.client.containers(all=True))

 service.options['environment']['FOO'] = '2'
 new_container, = service.recreate_containers()

 self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(new_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=2', new_container.dictionary['Config']['Env'])
 self.assertEqual(new_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(new_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=2', new_container.get('Config.Env'))
 self.assertEqual(new_container.name, 'composetest_db_1')
 self.assertEqual(new_container.inspect()['Volumes']['/etc'], volume_path)
 self.assertIn(old_container.id, new_container.dictionary['HostConfig']['VolumesFrom'])
 self.assertEqual(new_container.get('Volumes')['/etc'], volume_path)
 self.assertIn(
 'affinity:container==%s' % old_container.id,
 new_container.get('Config.Env'))

 self.assertEqual(len(self.client.containers(all=True)), num_containers_before)
 self.assertNotEqual(old_container.id, new_container.id)
Expand DownExpand Up@@ -289,9 +292,7 @@ def test_recreate_containers_with_image_declared_volume(self):
 self.assertEqual(old_container.get('Volumes').keys(), ['/data'])
 volume_path = old_container.get('Volumes')['/data']

 service.recreate_containers()
 new_container = service.containers()[0]
 service.start_container(new_container)
 new_container = service.recreate_containers()[0]
 self.assertEqual(new_container.get('Volumes').keys(), ['/data'])
 self.assertEqual(new_container.get('Volumes')['/data'], volume_path)

Expand Down

70 changes: 63 additions & 7 deletions
tests/unit/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,9 @@
 ConfigError,
 build_port_bindings,
 build_volume_binding,
 get_container_data_volumes,
 get_container_name,
 merge_volume_bindings,
 parse_repository_tag,
 parse_volume_spec,
 split_port,
Expand DownExpand Up@@ -86,13 +88,6 @@ def test_get_volumes_from_container(self):

 self.assertEqual(service._get_volumes_from(), [container_id])

 def test_get_volumes_from_previous_container(self):
 container_id = 'aabbccddee'
 service = Service('test', image='foo')
 container = mock.Mock(id=container_id, spec=Container, image='foo')

 self.assertEqual(service._get_volumes_from(container), [container_id])

 def test_get_volumes_from_service_container_exists(self):
 container_ids = ['aabbccddee', '12345']
 from_service = mock.create_autospec(Service)
Expand DownExpand Up@@ -320,6 +315,9 @@ def test_create_container_no_build(self):

class ServiceVolumesTest(unittest.TestCase):

 def setUp(self):
 self.mock_client = mock.create_autospec(docker.Client)

 def test_parse_volume_spec_only_one_path(self):
 spec = parse_volume_spec('/the/volume')
 self.assertEqual(spec, (None, '/the/volume', 'rw'))
Expand All@@ -345,3 +343,61 @@ def test_build_volume_binding(self):
 self.assertEqual(
 binding,
 ('/outside', dict(bind='/inside', ro=False)))

 def test_get_container_data_volumes(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {
 'Volumes': {
 '/mnt/image/data': {},
 }
 }
 }
 container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {
 '/host/volume': '/host/volume',
 '/existing/volume': '/var/lib/docker/aaaaaaaa',
 '/removed/volume': '/var/lib/docker/bbbbbbbb',
 '/mnt/image/data': '/var/lib/docker/cccccccc',
 },
 }, has_been_inspected=True)

 expected = {
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 '/var/lib/docker/cccccccc': {'bind': '/mnt/image/data', 'ro': False},
 }

 binds = get_container_data_volumes(container, options)
 self.assertEqual(binds, expected)

 def test_merge_volume_bindings(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/host/rw/volume:/host/rw/volume',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {'Volumes': {}}
 }

 intermediate_container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'},
 }, has_been_inspected=True)

 expected = {
 '/host/volume': {'bind': '/host/volume', 'ro': True},
 '/host/rw/volume': {'bind': '/host/rw/volume', 'ro': False},
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 }

 binds = merge_volume_bindings(options, intermediate_container)
 self.assertEqual(binds, expected)

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Preserve individual volumes on recreate by dnephin · Pull Request #858 · docker/compose · GitHub
Skip to content
Preserve individual volumes on recreate#858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

aanand
merged 1 commit into

from

May 14, 2015

+152

−40

 Merged


Preserve individual volumes on recreate
#858
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5 changes: 1 addition & 4 deletions
compose/cli/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,10 +317,7 @@ def run(self, project, options):
 }

 if options['-e']:
 # Merge environment from config with -e command line
 container_options['environment'] = dict(
 parse_environment(service.options.get('environment')),
 **parse_environment(options['-e']))
 container_options['environment'] = parse_environment(options['-e'])

 if options['--entrypoint']:
 container_options['entrypoint'] = options.get('--entrypoint')
Expand Down

4 changes: 4 additions & 0 deletions
compose/container.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -44,6 +44,10 @@ def id(self):
 def image(self):
 return self.dictionary['Image']

 @property
 def image_config(self):
 return self.client.inspect_image(self.image)

 @property
 def short_id(self):
 return self.id[:10]
Expand Down

86 changes: 70 additions & 16 deletions
compose/service.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,14 +3,14 @@
from collections import namedtuple
import logging
import re
from operator import attrgetter
import sys
import six
from operator import attrgetter

import six
from docker.errors import APIError
from docker.utils import create_host_config, LogConfig

from .config import DOCKER_CONFIG_KEYS
from .config import DOCKER_CONFIG_KEYS, merge_environment
from .container import Container, get_container_name
from .progress_stream import stream_output, StreamOutputError

Expand DownExpand Up@@ -183,10 +183,10 @@ def create_container(self,
 Create a container for this service. If the image doesn't exist, attempt to pull
 it.
 """
 override_options['volumes_from'] = self._get_volumes_from(previous_container)
 container_options = self._get_container_create_options(
 override_options,
 one_off=one_off,
 previous_container=previous_container,
 )

 if (do_build and
Expand DownExpand Up@@ -247,6 +247,12 @@ def recreate_container(self, container, **override_options):
 self.client.rename(
 container.id,
 '%s_%s' % (container.short_id, container.name))

 override_options = dict(
 override_options,
 environment=merge_environment(
 override_options.get('environment'),
 {'affinity:container': '=' + container.id}))
 new_container = self.create_container(
 do_build=False,
 previous_container=container,
Expand DownExpand Up@@ -326,7 +332,7 @@ def _get_links(self, link_to_self):
 links.append((external_link, link_name))
 return links

 def _get_volumes_from(self, previous_container=None):
 def _get_volumes_from(self):
 volumes_from = []
 for volume_source in self.volumes_from:
 if isinstance(volume_source, Service):
Expand All@@ -339,9 +345,6 @@ def _get_volumes_from(self, previous_container=None):
 elif isinstance(volume_source, Container):
 volumes_from.append(volume_source.id)

 if previous_container:
 volumes_from.append(previous_container.id)

 return volumes_from

 def _get_net(self):
Expand All@@ -363,7 +366,11 @@ def _get_net(self):

 return net

 def _get_container_create_options(self, override_options, one_off=False):
 def _get_container_create_options(
 self,
 override_options,
 one_off=False,
 previous_container=None):
 container_options = dict(
 (k, self.options[k])
 for k in DOCKER_CONFIG_KEYS if k in self.options)
Expand DownExpand Up@@ -396,11 +403,19 @@ def _get_container_create_options(self, override_options, one_off=False):
 ports.append(port)
 container_options['ports'] = ports

 override_options['binds'] = merge_volume_bindings(
 container_options.get('volumes') or [],
 previous_container)

 if 'volumes' in container_options:
 container_options['volumes'] = dict(
 (parse_volume_spec(v).internal, {})
 for v in container_options['volumes'])

 container_options['environment'] = merge_environment(
 self.options.get('environment'),
 override_options.get('environment'))

 if self.can_be_built():
 container_options['image'] = self.full_name

Expand All@@ -418,11 +433,6 @@ def _get_container_host_config(self, override_options, one_off=False):
 options = dict(self.options, **override_options)
 port_bindings = build_port_bindings(options.get('ports') or [])

 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in options.get('volumes') or []
 if ':' in volume)

 privileged = options.get('privileged', False)
 cap_add = options.get('cap_add', None)
 cap_drop = options.get('cap_drop', None)
Expand All@@ -447,8 +457,8 @@ def _get_container_host_config(self, override_options, one_off=False):
 return create_host_config(
 links=self._get_links(link_to_self=one_off),
 port_bindings=port_bindings,
 binds=volume_bindings,
 volumes_from=options.get('volumes_from'),
 binds=options.get('binds'),
 volumes_from=self._get_volumes_from(),
 privileged=privileged,
 network_mode=self._get_net(),
 devices=devices,
Expand DownExpand Up@@ -531,6 +541,50 @@ def pull(self, insecure_registry=False):
 stream_output(output, sys.stdout)


def get_container_data_volumes(container, volumes_option):
 """Find the container data volumes that are in `volumes_option`, and return
 a mapping of volume bindings for those volumes.
 """
 volumes = []

 volumes_option = volumes_option or []
 container_volumes = container.get('Volumes') or {}
 image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {}

 for volume in set(volumes_option + image_volumes.keys()):
 volume = parse_volume_spec(volume)
 # No need to preserve host volumes
 if volume.external:
 continue

 volume_path = container_volumes.get(volume.internal)
 # New volume, doesn't exist in the old container
 if not volume_path:
 continue

 # Copy existing volume from old container
 volume = volume._replace(external=volume_path)
 volumes.append(build_volume_binding(volume))

 return dict(volumes)


def merge_volume_bindings(volumes_option, previous_container):
 """Return a list of volume bindings for a container. Container data volumes
 are replaced by those from the previous container.
 """
 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in volumes_option or []
 if ':' in volume)

 if previous_container:
 volume_bindings.update(
 get_container_data_volumes(previous_container, volumes_option))

 return volume_bindings


NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')


Expand Down

27 changes: 14 additions & 13 deletions
tests/integration/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def test_create_container_with_unspecified_volume(self):
 service = self.create_service('db', volumes=['/var/db'])
 container = service.create_container()
 service.start_container(container)
 self.assertIn('/var/db', container.inspect()['Volumes'])
 self.assertIn('/var/db', container.get('Volumes'))

 def test_create_container_with_cpu_shares(self):
 service = self.create_service('db', cpu_shares=73)
Expand DownExpand Up@@ -239,24 +239,27 @@ def test_recreate_containers(self):
 command=['300']
 )
 old_container = service.create_container()
 self.assertEqual(old_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(old_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=1', old_container.dictionary['Config']['Env'])
 self.assertEqual(old_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(old_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=1', old_container.get('Config.Env'))
 self.assertEqual(old_container.name, 'composetest_db_1')
 service.start_container(old_container)
 volume_path = old_container.inspect()['Volumes']['/etc']
 old_container.inspect() # reload volume data
 volume_path = old_container.get('Volumes')['/etc']

 num_containers_before = len(self.client.containers(all=True))

 service.options['environment']['FOO'] = '2'
 new_container, = service.recreate_containers()

 self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(new_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=2', new_container.dictionary['Config']['Env'])
 self.assertEqual(new_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(new_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=2', new_container.get('Config.Env'))
 self.assertEqual(new_container.name, 'composetest_db_1')
 self.assertEqual(new_container.inspect()['Volumes']['/etc'], volume_path)
 self.assertIn(old_container.id, new_container.dictionary['HostConfig']['VolumesFrom'])
 self.assertEqual(new_container.get('Volumes')['/etc'], volume_path)
 self.assertIn(
 'affinity:container==%s' % old_container.id,
 new_container.get('Config.Env'))

 self.assertEqual(len(self.client.containers(all=True)), num_containers_before)
 self.assertNotEqual(old_container.id, new_container.id)
Expand DownExpand Up@@ -289,9 +292,7 @@ def test_recreate_containers_with_image_declared_volume(self):
 self.assertEqual(old_container.get('Volumes').keys(), ['/data'])
 volume_path = old_container.get('Volumes')['/data']

 service.recreate_containers()
 new_container = service.containers()[0]
 service.start_container(new_container)
 new_container = service.recreate_containers()[0]
 self.assertEqual(new_container.get('Volumes').keys(), ['/data'])
 self.assertEqual(new_container.get('Volumes')['/data'], volume_path)

Expand Down

70 changes: 63 additions & 7 deletions
tests/unit/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,9 @@
 ConfigError,
 build_port_bindings,
 build_volume_binding,
 get_container_data_volumes,
 get_container_name,
 merge_volume_bindings,
 parse_repository_tag,
 parse_volume_spec,
 split_port,
Expand DownExpand Up@@ -86,13 +88,6 @@ def test_get_volumes_from_container(self):

 self.assertEqual(service._get_volumes_from(), [container_id])

 def test_get_volumes_from_previous_container(self):
 container_id = 'aabbccddee'
 service = Service('test', image='foo')
 container = mock.Mock(id=container_id, spec=Container, image='foo')

 self.assertEqual(service._get_volumes_from(container), [container_id])

 def test_get_volumes_from_service_container_exists(self):
 container_ids = ['aabbccddee', '12345']
 from_service = mock.create_autospec(Service)
Expand DownExpand Up@@ -320,6 +315,9 @@ def test_create_container_no_build(self):

class ServiceVolumesTest(unittest.TestCase):

 def setUp(self):
 self.mock_client = mock.create_autospec(docker.Client)

 def test_parse_volume_spec_only_one_path(self):
 spec = parse_volume_spec('/the/volume')
 self.assertEqual(spec, (None, '/the/volume', 'rw'))
Expand All@@ -345,3 +343,61 @@ def test_build_volume_binding(self):
 self.assertEqual(
 binding,
 ('/outside', dict(bind='/inside', ro=False)))

 def test_get_container_data_volumes(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {
 'Volumes': {
 '/mnt/image/data': {},
 }
 }
 }
 container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {
 '/host/volume': '/host/volume',
 '/existing/volume': '/var/lib/docker/aaaaaaaa',
 '/removed/volume': '/var/lib/docker/bbbbbbbb',
 '/mnt/image/data': '/var/lib/docker/cccccccc',
 },
 }, has_been_inspected=True)

 expected = {
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 '/var/lib/docker/cccccccc': {'bind': '/mnt/image/data', 'ro': False},
 }

 binds = get_container_data_volumes(container, options)
 self.assertEqual(binds, expected)

 def test_merge_volume_bindings(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/host/rw/volume:/host/rw/volume',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {'Volumes': {}}
 }

 intermediate_container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'},
 }, has_been_inspected=True)

 expected = {
 '/host/volume': {'bind': '/host/volume', 'ro': True},
 '/host/rw/volume': {'bind': '/host/rw/volume', 'ro': False},
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 }

 binds = merge_volume_bindings(options, intermediate_container)
 self.assertEqual(binds, expected)

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
Preserve individual volumes on recreate by dnephin · Pull Request #858 · docker/compose · GitHub
Skip to content
Preserve individual volumes on recreate#858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

aanand
merged 1 commit into

from

May 14, 2015

+152

−40

 Merged


Preserve individual volumes on recreate
#858
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5 changes: 1 addition & 4 deletions
compose/cli/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -317,10 +317,7 @@ def run(self, project, options):
 }

 if options['-e']:
 # Merge environment from config with -e command line
 container_options['environment'] = dict(
 parse_environment(service.options.get('environment')),
 **parse_environment(options['-e']))
 container_options['environment'] = parse_environment(options['-e'])

 if options['--entrypoint']:
 container_options['entrypoint'] = options.get('--entrypoint')
Expand Down

4 changes: 4 additions & 0 deletions
compose/container.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -44,6 +44,10 @@ def id(self):
 def image(self):
 return self.dictionary['Image']

 @property
 def image_config(self):
 return self.client.inspect_image(self.image)

 @property
 def short_id(self):
 return self.id[:10]
Expand Down

86 changes: 70 additions & 16 deletions
compose/service.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,14 +3,14 @@
from collections import namedtuple
import logging
import re
from operator import attrgetter
import sys
import six
from operator import attrgetter

import six
from docker.errors import APIError
from docker.utils import create_host_config, LogConfig

from .config import DOCKER_CONFIG_KEYS
from .config import DOCKER_CONFIG_KEYS, merge_environment
from .container import Container, get_container_name
from .progress_stream import stream_output, StreamOutputError

Expand DownExpand Up@@ -183,10 +183,10 @@ def create_container(self,
 Create a container for this service. If the image doesn't exist, attempt to pull
 it.
 """
 override_options['volumes_from'] = self._get_volumes_from(previous_container)
 container_options = self._get_container_create_options(
 override_options,
 one_off=one_off,
 previous_container=previous_container,
 )

 if (do_build and
Expand DownExpand Up@@ -247,6 +247,12 @@ def recreate_container(self, container, **override_options):
 self.client.rename(
 container.id,
 '%s_%s' % (container.short_id, container.name))

 override_options = dict(
 override_options,
 environment=merge_environment(
 override_options.get('environment'),
 {'affinity:container': '=' + container.id}))
 new_container = self.create_container(
 do_build=False,
 previous_container=container,
Expand DownExpand Up@@ -326,7 +332,7 @@ def _get_links(self, link_to_self):
 links.append((external_link, link_name))
 return links

 def _get_volumes_from(self, previous_container=None):
 def _get_volumes_from(self):
 volumes_from = []
 for volume_source in self.volumes_from:
 if isinstance(volume_source, Service):
Expand All@@ -339,9 +345,6 @@ def _get_volumes_from(self, previous_container=None):
 elif isinstance(volume_source, Container):
 volumes_from.append(volume_source.id)

 if previous_container:
 volumes_from.append(previous_container.id)

 return volumes_from

 def _get_net(self):
Expand All@@ -363,7 +366,11 @@ def _get_net(self):

 return net

 def _get_container_create_options(self, override_options, one_off=False):
 def _get_container_create_options(
 self,
 override_options,
 one_off=False,
 previous_container=None):
 container_options = dict(
 (k, self.options[k])
 for k in DOCKER_CONFIG_KEYS if k in self.options)
Expand DownExpand Up@@ -396,11 +403,19 @@ def _get_container_create_options(self, override_options, one_off=False):
 ports.append(port)
 container_options['ports'] = ports

 override_options['binds'] = merge_volume_bindings(
 container_options.get('volumes') or [],
 previous_container)

 if 'volumes' in container_options:
 container_options['volumes'] = dict(
 (parse_volume_spec(v).internal, {})
 for v in container_options['volumes'])

 container_options['environment'] = merge_environment(
 self.options.get('environment'),
 override_options.get('environment'))

 if self.can_be_built():
 container_options['image'] = self.full_name

Expand All@@ -418,11 +433,6 @@ def _get_container_host_config(self, override_options, one_off=False):
 options = dict(self.options, **override_options)
 port_bindings = build_port_bindings(options.get('ports') or [])

 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in options.get('volumes') or []
 if ':' in volume)

 privileged = options.get('privileged', False)
 cap_add = options.get('cap_add', None)
 cap_drop = options.get('cap_drop', None)
Expand All@@ -447,8 +457,8 @@ def _get_container_host_config(self, override_options, one_off=False):
 return create_host_config(
 links=self._get_links(link_to_self=one_off),
 port_bindings=port_bindings,
 binds=volume_bindings,
 volumes_from=options.get('volumes_from'),
 binds=options.get('binds'),
 volumes_from=self._get_volumes_from(),
 privileged=privileged,
 network_mode=self._get_net(),
 devices=devices,
Expand DownExpand Up@@ -531,6 +541,50 @@ def pull(self, insecure_registry=False):
 stream_output(output, sys.stdout)


def get_container_data_volumes(container, volumes_option):
 """Find the container data volumes that are in `volumes_option`, and return
 a mapping of volume bindings for those volumes.
 """
 volumes = []

 volumes_option = volumes_option or []
 container_volumes = container.get('Volumes') or {}
 image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {}

 for volume in set(volumes_option + image_volumes.keys()):
 volume = parse_volume_spec(volume)
 # No need to preserve host volumes
 if volume.external:
 continue

 volume_path = container_volumes.get(volume.internal)
 # New volume, doesn't exist in the old container
 if not volume_path:
 continue

 # Copy existing volume from old container
 volume = volume._replace(external=volume_path)
 volumes.append(build_volume_binding(volume))

 return dict(volumes)


def merge_volume_bindings(volumes_option, previous_container):
 """Return a list of volume bindings for a container. Container data volumes
 are replaced by those from the previous container.
 """
 volume_bindings = dict(
 build_volume_binding(parse_volume_spec(volume))
 for volume in volumes_option or []
 if ':' in volume)

 if previous_container:
 volume_bindings.update(
 get_container_data_volumes(previous_container, volumes_option))

 return volume_bindings


NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')


Expand Down

27 changes: 14 additions & 13 deletions
tests/integration/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,7 @@ def test_create_container_with_unspecified_volume(self):
 service = self.create_service('db', volumes=['/var/db'])
 container = service.create_container()
 service.start_container(container)
 self.assertIn('/var/db', container.inspect()['Volumes'])
 self.assertIn('/var/db', container.get('Volumes'))

 def test_create_container_with_cpu_shares(self):
 service = self.create_service('db', cpu_shares=73)
Expand DownExpand Up@@ -239,24 +239,27 @@ def test_recreate_containers(self):
 command=['300']
 )
 old_container = service.create_container()
 self.assertEqual(old_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(old_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=1', old_container.dictionary['Config']['Env'])
 self.assertEqual(old_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(old_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=1', old_container.get('Config.Env'))
 self.assertEqual(old_container.name, 'composetest_db_1')
 service.start_container(old_container)
 volume_path = old_container.inspect()['Volumes']['/etc']
 old_container.inspect() # reload volume data
 volume_path = old_container.get('Volumes')['/etc']

 num_containers_before = len(self.client.containers(all=True))

 service.options['environment']['FOO'] = '2'
 new_container, = service.recreate_containers()

 self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['sleep'])
 self.assertEqual(new_container.dictionary['Config']['Cmd'], ['300'])
 self.assertIn('FOO=2', new_container.dictionary['Config']['Env'])
 self.assertEqual(new_container.get('Config.Entrypoint'), ['sleep'])
 self.assertEqual(new_container.get('Config.Cmd'), ['300'])
 self.assertIn('FOO=2', new_container.get('Config.Env'))
 self.assertEqual(new_container.name, 'composetest_db_1')
 self.assertEqual(new_container.inspect()['Volumes']['/etc'], volume_path)
 self.assertIn(old_container.id, new_container.dictionary['HostConfig']['VolumesFrom'])
 self.assertEqual(new_container.get('Volumes')['/etc'], volume_path)
 self.assertIn(
 'affinity:container==%s' % old_container.id,
 new_container.get('Config.Env'))

 self.assertEqual(len(self.client.containers(all=True)), num_containers_before)
 self.assertNotEqual(old_container.id, new_container.id)
Expand DownExpand Up@@ -289,9 +292,7 @@ def test_recreate_containers_with_image_declared_volume(self):
 self.assertEqual(old_container.get('Volumes').keys(), ['/data'])
 volume_path = old_container.get('Volumes')['/data']

 service.recreate_containers()
 new_container = service.containers()[0]
 service.start_container(new_container)
 new_container = service.recreate_containers()[0]
 self.assertEqual(new_container.get('Volumes').keys(), ['/data'])
 self.assertEqual(new_container.get('Volumes')['/data'], volume_path)

Expand Down

70 changes: 63 additions & 7 deletions
tests/unit/service_test.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,9 @@
 ConfigError,
 build_port_bindings,
 build_volume_binding,
 get_container_data_volumes,
 get_container_name,
 merge_volume_bindings,
 parse_repository_tag,
 parse_volume_spec,
 split_port,
Expand DownExpand Up@@ -86,13 +88,6 @@ def test_get_volumes_from_container(self):

 self.assertEqual(service._get_volumes_from(), [container_id])

 def test_get_volumes_from_previous_container(self):
 container_id = 'aabbccddee'
 service = Service('test', image='foo')
 container = mock.Mock(id=container_id, spec=Container, image='foo')

 self.assertEqual(service._get_volumes_from(container), [container_id])

 def test_get_volumes_from_service_container_exists(self):
 container_ids = ['aabbccddee', '12345']
 from_service = mock.create_autospec(Service)
Expand DownExpand Up@@ -320,6 +315,9 @@ def test_create_container_no_build(self):

class ServiceVolumesTest(unittest.TestCase):

 def setUp(self):
 self.mock_client = mock.create_autospec(docker.Client)

 def test_parse_volume_spec_only_one_path(self):
 spec = parse_volume_spec('/the/volume')
 self.assertEqual(spec, (None, '/the/volume', 'rw'))
Expand All@@ -345,3 +343,61 @@ def test_build_volume_binding(self):
 self.assertEqual(
 binding,
 ('/outside', dict(bind='/inside', ro=False)))

 def test_get_container_data_volumes(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {
 'Volumes': {
 '/mnt/image/data': {},
 }
 }
 }
 container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {
 '/host/volume': '/host/volume',
 '/existing/volume': '/var/lib/docker/aaaaaaaa',
 '/removed/volume': '/var/lib/docker/bbbbbbbb',
 '/mnt/image/data': '/var/lib/docker/cccccccc',
 },
 }, has_been_inspected=True)

 expected = {
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 '/var/lib/docker/cccccccc': {'bind': '/mnt/image/data', 'ro': False},
 }

 binds = get_container_data_volumes(container, options)
 self.assertEqual(binds, expected)

 def test_merge_volume_bindings(self):
 options = [
 '/host/volume:/host/volume:ro',
 '/host/rw/volume:/host/rw/volume',
 '/new/volume',
 '/existing/volume',
 ]

 self.mock_client.inspect_image.return_value = {
 'ContainerConfig': {'Volumes': {}}
 }

 intermediate_container = Container(self.mock_client, {
 'Image': 'ababab',
 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'},
 }, has_been_inspected=True)

 expected = {
 '/host/volume': {'bind': '/host/volume', 'ro': True},
 '/host/rw/volume': {'bind': '/host/rw/volume', 'ro': False},
 '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False},
 }

 binds = merge_volume_bindings(options, intermediate_container)
 self.assertEqual(binds, expected)