Skip to content

Commit

Permalink
slobjects can not be made of other slobjects
Browse files Browse the repository at this point in the history 
Change-Id: Ib500f5fc2e890c3d23641350815a910168bcd543
  • Loading branch information
@dpgoetz
dpgoetz committed Apr 12, 2013
1 parent c496ba1 commit 3f21e14
Show file tree
Hide file tree
Showing 5 changed files with 92 additions and 18 deletions.
4 changes: 4 additions & 0 deletions swift/common/middleware/slo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,10 @@ def handle_multipart_put(self, req):
problem_segments.append([quote(obj_path), 'Size Mismatch'])
if seg_dict['etag'] != head_seg_resp.etag:
problem_segments.append([quote(obj_path), 'Etag Mismatch'])
if 'X-Static-Large-Object' in head_seg_resp.headers or \
'X-Object-Manifest' in head_seg_resp.headers:
problem_segments.append(
[quote(obj_path), 'Segments cannot be Large Objects'])
if head_seg_resp.last_modified:
last_modified = head_seg_resp.last_modified
else:
Expand Down
3 changes: 3 additions & 0 deletions swift/proxy/controllers/obj.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,9 @@ def _load_next_segment(self):
'%(path)s etag: %(r_etag)s != %(s_etag)s.' %
{'path': path, 'r_etag': resp.etag,
's_etag': self.segment_dict['hash']}))
if 'X-Static-Large-Object' in resp.headers:
raise SloSegmentError(_(
'SLO can not be made of other SLOs: %s' % path))
self.segment_iter = resp.app_iter
# See NOTE: swift_conn at top of file about this.
self.segment_iter_swift_conn = getattr(resp, 'swift_conn', None)
Expand Down
14 changes: 10 additions & 4 deletions test/unit/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ def fake_http_connect(*code_iter, **kwargs):
class FakeConn(object):

def __init__(self, status, etag=None, body='', timestamp='1',
expect_status=None):
expect_status=None, headers=None):
self.status = status
if expect_status is None:
self.expect_status = self.status
Expand All @@ -286,6 +286,7 @@ def __init__(self, status, etag=None, body='', timestamp='1',
self.received = 0
self.etag = etag
self.body = body
self.headers = headers or {}
self.timestamp = timestamp

def getresponse(self):
Expand Down Expand Up @@ -329,8 +330,7 @@ def getheaders(self):
pass
if 'slow' in kwargs:
headers['content-length'] = '4'
if 'headers' in kwargs:
headers.update(kwargs['headers'])
headers.update(self.headers)
return headers.items()

def read(self, amt=None):
Expand All @@ -354,6 +354,11 @@ def getheader(self, name, default=None):

timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter))
etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter))
if isinstance(kwargs.get('headers'), list):
headers_iter = iter(kwargs['headers'])
else:
headers_iter = iter([kwargs.get('headers', {})] * len(code_iter))

x = kwargs.get('missing_container', [False] * len(code_iter))
if not isinstance(x, (tuple, list)):
x = [x] * len(code_iter)
Expand All @@ -378,6 +383,7 @@ def connect(*args, **ckwargs):
else:
expect_status = status
etag = etag_iter.next()
headers = headers_iter.next()
timestamp = timestamps_iter.next()

if status <= 0:
Expand All @@ -387,6 +393,6 @@ def connect(*args, **ckwargs):
else:
body = body_iter.next()
return FakeConn(status, etag, body=body, timestamp=timestamp,
expect_status=expect_status)
expect_status=expect_status, headers=headers)

return connect
21 changes: 13 additions & 8 deletions test/unit/common/middleware/test_slo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,11 @@ def __call__(self, env, start_response):
cont_len = 100
if obj == 'small_object':
cont_len = 10
return Response(
status=200,
headers={'etag': 'etagoftheobjectsegment',
'Content-Length': cont_len})(env, start_response)
headers = {'etag': 'etagoftheobjectsegment',
'Content-Length': cont_len}
if obj == 'slob':
headers['X-Static-Large-Object'] = 'true'
return Response(status=200, headers=headers)(env, start_response)
if env['PATH_INFO'].startswith('/test_good_check/'):
j, v, a, cont, obj = env['PATH_INFO'].split('/')
etag, size = obj.split('_')
Expand Down Expand Up @@ -307,7 +308,8 @@ def test_handle_multipart_put_check_data_bad(self):
bad_data = json.dumps(
[{'path': '/c/a_1', 'etag': 'a', 'size_bytes': '1'},
{'path': '/c/a_2', 'etag': 'a', 'size_bytes': '1'},
{'path': '/d/b_2', 'etag': 'b', 'size_bytes': '2'}])
{'path': '/d/b_2', 'etag': 'b', 'size_bytes': '2'},
{'path': '/d/slob', 'etag': 'b', 'size_bytes': '2'}])
req = Request.blank(
'/test_good/A/c/man?multipart-manifest=put',
environ={'REQUEST_METHOD': 'PUT'},
Expand All @@ -316,14 +318,17 @@ def test_handle_multipart_put_check_data_bad(self):
try:
self.slo.handle_multipart_put(req)
except HTTPException, e:
self.assertEquals(self.app.calls, 3)
self.assertEquals(self.app.calls, 4)
data = json.loads(e.body)
errors = data['Errors']
self.assertEquals(errors[0][0], '/test_good/A/c/a_1')
self.assertEquals(errors[0][1], 'Size Mismatch')
self.assertEquals(errors[2][1], '400 Bad Request')
self.assertEquals(errors[-1][0], '/test_good/A/d/b_2')
self.assertEquals(errors[-1][1], 'Etag Mismatch')
self.assertEquals(errors[4][0], '/test_good/A/d/b_2')
self.assertEquals(errors[4][1], 'Etag Mismatch')
self.assertEquals(errors[-1][0], '/test_good/A/d/slob')
self.assertEquals(errors[-1][1],
'Segments cannot be Large Objects')
else:
self.assert_(False)

Expand Down
68 changes: 62 additions & 6 deletions test/unit/proxy/test_server.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1021,8 +1021,8 @@ def capture_requested_paths(ipaddr, port, device, partition,
200, # GET listing1
200, # GET seg01
200, # GET seg02
headers={"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'},
headers=[{}, {}, {"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'}, {}, {}],
body_iter=response_bodies,
give_connect=capture_requested_paths)

Expand Down Expand Up @@ -1176,8 +1176,64 @@ def capture_requested_paths(ipaddr, port, device, partition,
200, # GET listing1
200, # GET seg01
200, # GET seg02
headers={"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'},
headers=[{}, {}, {"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'}, {}, {}],
body_iter=response_bodies,
give_connect=capture_requested_paths)
req = Request.blank('/a/c/manifest')
resp = controller.GET(req)
self.assertEqual(resp.status_int, 200)
self.assertEqual(resp.body, 'Aa') # dropped connection
self.assertEqual(resp.content_length, 4) # content incomplete
self.assertEqual(resp.content_type, 'text/html')

self.assertEqual(
requested,
[['HEAD', '/a', {}],
['HEAD', '/a/c', {}],
['GET', '/a/c/manifest', {}],
['GET', '/a/d1/seg01', {}],
['GET', '/a/d2/seg02', {}]])

def test_GET_nested_slo(self):
listing = [{"hash": "98568d540134639be4655198a36614a4",
"last_modified": "2012-11-08T04:05:37.866820",
"bytes": 2,
"name": "/d1/seg01",
"content_type": "application/octet-stream"},
{"hash": "d526f1c8ef6c1e4e980e2b8471352d23",
"last_modified": "2012-11-08T04:05:37.846710",
"bytes": 2,
"name": "/d2/seg02",
"content_type": "application/octet-stream"}]

response_bodies = (
'', # HEAD /a
'', # HEAD /a/c
simplejson.dumps(listing), # GET manifest
'Aa', # GET seg01
'Bb') # GET seg02
with save_globals():
controller = proxy_server.ObjectController(
self.app, 'a', 'c', 'manifest')

requested = []

def capture_requested_paths(ipaddr, port, device, partition,
method, path, headers=None,
query_string=None):
qs_dict = dict(urlparse.parse_qsl(query_string or ''))
requested.append([method, path, qs_dict])

slob_headers = {"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'}
set_http_connect(
200, # HEAD /a
200, # HEAD /a/c
200, # GET listing1
200, # GET seg01
200, # GET seg02
headers=[{}, {}, slob_headers, {}, slob_headers],
body_iter=response_bodies,
give_connect=capture_requested_paths)
req = Request.blank('/a/c/manifest')
Expand Down Expand Up @@ -1236,8 +1292,8 @@ def capture_requested_paths(ipaddr, port, device, partition,
200, # GET listing1
200, # GET seg01
404, # GET seg02
headers={"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'},
headers=[{}, {}, {"X-Static-Large-Object": "True",
'content-type': 'text/html; swift_bytes=4'}, {}, {}],
body_iter=response_bodies,
give_connect=capture_requested_paths)
req = Request.blank('/a/c/manifest')
Expand Down

0 comments on commit 3f21e14

Please sign in to comment.