From ce6b4809b31765316277672a0abc2f3c12b04a10 Mon Sep 17 00:00:00 2001 From: Ian Cottrell Date: Wed, 17 Jun 2015 14:50:43 +0100 Subject: Clean up Blob handling This is changes extracted from 155130 to make it more discussable. Change-Id: I91e9cd1a46505906a27ca01a41ed00b53c6e42c7 --- database/blob.go | 42 ++++++ database/database_binary.go | 60 +++++++++ database/store/blob.go | 24 ---- database/store/cache_test.go | 287 ----------------------------------------- database/store/store_binary.go | 60 --------- 5 files changed, 102 insertions(+), 371 deletions(-) create mode 100644 database/blob.go delete mode 100644 database/store/blob.go delete mode 100644 database/store/cache_test.go (limited to 'database') diff --git a/database/blob.go b/database/blob.go new file mode 100644 index 000000000..1c43c30b2 --- /dev/null +++ b/database/blob.go @@ -0,0 +1,42 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "android.googlesource.com/platform/tools/gpu/binary" + "android.googlesource.com/platform/tools/gpu/log" +) + +// blob is an encodable wrapper for a byte array, used for storing raw data +// in databases. +type blob struct { + binary.Generate + Data []byte +} + +// StoreBlob stores the byte slice data inside a Blob to the database d. +func StoreBlob(data []byte, d Database, l log.Logger) (binary.ID, error) { + return d.Store(&blob{Data: data}, l) +} + +// Resolve blob loads a Blob from the database, returning the byte slice. +func ResolveBlob(id binary.ID, d Database, l log.Logger) ([]byte, error) { + b := blob{} + err := d.Load(id, l, &b) + if err != nil { + return nil, err + } + return b.Data, nil +} diff --git a/database/database_binary.go b/database/database_binary.go index bb5d764ac..b0fd6f4a0 100644 --- a/database/database_binary.go +++ b/database/database_binary.go @@ -12,13 +12,73 @@ import ( ) func init() { + registry.Add((*blob)(nil).Class()) registry.Add((*metadata)(nil).Class()) } var ( + binaryIDblob = binary.ID{0x38, 0x16, 0x87, 0x7a, 0x38, 0x4f, 0xaf, 0x5d, 0x34, 0xf4, 0xeb, 0x7e, 0x3f, 0x26, 0x23, 0x3d, 0x6f, 0xd8, 0x32, 0x62} binaryIDmetadata = binary.ID{0x84, 0x31, 0x02, 0x95, 0x2a, 0x0a, 0x75, 0xc0, 0x5a, 0xe3, 0x0b, 0x4c, 0x25, 0x31, 0xa9, 0x0f, 0x5e, 0xf6, 0xfd, 0x35} ) +type binaryClassblob struct{} + +func (*blob) Class() binary.Class { + return (*binaryClassblob)(nil) +} +func doEncodeblob(e binary.Encoder, o *blob) error { + if err := e.Uint32(uint32(len(o.Data))); err != nil { + return err + } + if err := e.Data(o.Data); err != nil { + return err + } + return nil +} +func doDecodeblob(d binary.Decoder, o *blob) error { + if count, err := d.Uint32(); err != nil { + return err + } else { + o.Data = make([]byte, count) + if err := d.Data(o.Data); err != nil { + return err + } + } + return nil +} +func doSkipblob(d binary.Decoder) error { + if count, err := d.Uint32(); err != nil { + return err + } else { + if err := d.Skip(count); err != nil { + return err + } + } + return nil +} +func (*binaryClassblob) ID() binary.ID { return binaryIDblob } +func (*binaryClassblob) New() binary.Object { return &blob{} } +func (*binaryClassblob) Encode(e binary.Encoder, obj binary.Object) error { + return doEncodeblob(e, obj.(*blob)) +} +func (*binaryClassblob) Decode(d binary.Decoder) (binary.Object, error) { + obj := &blob{} + return obj, doDecodeblob(d, obj) +} +func (*binaryClassblob) DecodeTo(d binary.Decoder, obj binary.Object) error { + return doDecodeblob(d, obj.(*blob)) +} +func (*binaryClassblob) Skip(d binary.Decoder) error { return doSkipblob(d) } +func (*binaryClassblob) Schema() *schema.Class { return schemablob } + +var schemablob = &schema.Class{ + TypeID: binaryIDblob, + Name: "blob", + Fields: []schema.Field{ + {Declared: "Data", Type: &schema.Slice{Alias: "", ValueType: &schema.Primitive{Name: "byte", Method: schema.Uint8}}}, + }, +} + type binaryClassmetadata struct{} func (*metadata) Class() binary.Class { diff --git a/database/store/blob.go b/database/store/blob.go deleted file mode 100644 index cc7060a07..000000000 --- a/database/store/blob.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (C) 2015 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package store - -import "android.googlesource.com/platform/tools/gpu/binary" - -// Blob is an encodable wrapper for a byte array, used for storing raw data -// in databases. -type Blob struct { - binary.Generate - Data []byte -} diff --git a/database/store/cache_test.go b/database/store/cache_test.go deleted file mode 100644 index 421a9e5eb..000000000 --- a/database/store/cache_test.go +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright (C) 2015 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package store - -import ( - "bytes" - "reflect" - "testing" - - "android.googlesource.com/platform/tools/gpu/binary" - "android.googlesource.com/platform/tools/gpu/binary/cyclic" - "android.googlesource.com/platform/tools/gpu/binary/vle" - "android.googlesource.com/platform/tools/gpu/log" -) - -type storeCallback func(id binary.ID, r binary.Object, d []byte) error -type loadCallback func(id binary.ID, out binary.Object) (size int, err error) -type containsCallback func(id binary.ID) bool -type closeCallback func() - -type mockStore struct { - onStore storeCallback - onLoad loadCallback - onContains containsCallback - onClose closeCallback -} - -func (s mockStore) Store(id binary.ID, r binary.Object, d []byte, _ log.Logger) error { - return s.onStore(id, r, d) -} -func (s mockStore) Load(id binary.ID, _ log.Logger, out binary.Object) (size int, err error) { - return s.onLoad(id, out) -} -func (s mockStore) Contains(id binary.ID) bool { - return s.onContains(id) -} -func (s mockStore) Close() { - s.onClose() -} - -func createMockStore() *mockStore { - return &mockStore{ - onStore: func(binary.ID, binary.Object, []byte) error { return nil }, - onLoad: func(binary.ID, binary.Object) (size int, err error) { return -1, nil }, - onContains: func(binary.ID) bool { return false }, - onClose: func() {}, - } -} - -func validateLoad(t *testing.T, cache *cache, id binary.ID, expectedSize int, expectedData *Blob, expectedErr error) { - data := &Blob{Data: []byte{}} - size, err := cache.Load(id, nil, data) - if expectedSize != size { - t.Errorf("invalid size, expected %v got %v", expectedSize, size) - } - if !reflect.DeepEqual(expectedData, data) { - t.Errorf("invalid data, expected %v got %v", expectedData, data) - } - if expectedErr != err { - t.Errorf("invalid error, expected %v got %v", expectedErr, err) - } -} - -func encode(r binary.Object) []byte { - buf := &bytes.Buffer{} - enc := cyclic.Encoder(vle.Writer(buf)) - if err := enc.Value(r); err != nil { - panic(err) - } - return buf.Bytes() -} - -func calcSize(r binary.Object) int { - return len(encode(r)) -} - -func TestCacheMiss(t *testing.T) { - id := binary.NewID([]byte("123")) - data := &Blob{Data: []byte{1, 2, 3}} - size := calcSize(data) - loadCallCount := 0 - - inner := createMockStore() - inner.onLoad = func(actualId binary.ID, out binary.Object) (int, error) { - if id != actualId { - t.Errorf("invalid id, expected %v got %v", id, actualId) - } - loadCallCount++ - CopyResource(out, data) - return size, nil - } - - cache := CreateCache(64, inner).(*cache) - - // Nothing should have called inner.Load() yet - if 0 != loadCallCount { - t.Errorf("invalid load call count, expected %v got %v", 0, loadCallCount) - } - - validateLoad(t, cache, id, size, data, nil) - - // The cache should have missed, calling inner.Load() - if 1 != loadCallCount { - t.Errorf("invalid load call count, expected %v got %v", 1, loadCallCount) - } -} - -func TestCacheHit(t *testing.T) { - id := binary.NewID([]byte("123")) - data := &Blob{Data: []byte{1, 2, 3}} - size := calcSize(data) - loadCallCount := 0 - - inner := createMockStore() - inner.onLoad = func(actualId binary.ID, out binary.Object) (int, error) { - if id != actualId { - t.Errorf("invalid id, expected %v got %v", id, actualId) - } - loadCallCount++ - CopyResource(out, data) - return size, nil - } - - cache := CreateCache(64, inner).(*cache) - - // Warm the cache - validateLoad(t, cache, id, size, data, nil) - - // The cache should have hit, skipping the call into inner.Load() - validateLoad(t, cache, id, size, data, nil) - if 1 != loadCallCount { - t.Errorf("invalid load call count, expected %v got %v", 1, loadCallCount) - } -} - -func TestCacheMultipleLoads(t *testing.T) { - id := binary.NewID([]byte("123")) - data := &Blob{Data: []byte{1, 2, 3}} - size := calcSize(data) - loadCallCount := 0 - loadSync := make(chan bool) - loadUnblock := make(chan bool) - loadComplete := make(chan bool) - - inner := createMockStore() - inner.onLoad = func(actualId binary.ID, out binary.Object) (int, error) { - if id != actualId { - t.Errorf("invalid id, expected %v got %v", id, actualId) - } - loadCallCount++ - loadSync <- true - <-loadUnblock - CopyResource(out, data) - return size, nil - } - - cache := CreateCache(64, inner).(*cache) - - checkLoad := func() { - validateLoad(t, cache, id, size, data, nil) - loadComplete <- true - } - - // Begin a number of loads, but don't let the inner load finish just yet. - go checkLoad() - go checkLoad() - go checkLoad() - - // Sync with the inner load so we can query the load call count - <-loadSync - - // There should only have been one call to the inner load - if 1 != loadCallCount { - t.Errorf("invalid load call count, expected %v got %v", 1, loadCallCount) - } - - // Unblock the inner load - loadUnblock <- true - - // The three load requests should now finish - <-loadComplete - <-loadComplete - <-loadComplete - - // There still should only have been one call to the inner load - if 1 != loadCallCount { - t.Errorf("invalid load call count, expected %v got %v", 1, loadCallCount) - } -} - -func TestCacheOverflow(t *testing.T) { - idA, idB, idC := binary.NewID([]byte("123")), binary.NewID([]byte("456")), binary.NewID([]byte("789")) - dataA := &Blob{Data: []byte{1, 2, 3, 4}} - sizeA := calcSize(dataA) - dataB := &Blob{Data: []byte{4, 5, 6, 7}} - sizeB := calcSize(dataB) - dataC := &Blob{Data: []byte{8, 9, 10, 11}} - sizeC := calcSize(dataC) - loadCallCount := 0 - - inner := createMockStore() - inner.onLoad = func(id binary.ID, out binary.Object) (int, error) { - loadCallCount++ - switch id { - case idA: - CopyResource(out, dataA) - return sizeA, nil - case idB: - CopyResource(out, dataB) - return sizeB, nil - case idC: - CopyResource(out, dataC) - return sizeC, nil - default: - panic("Unexpected id!") - } - } - - // Create a cache that only has room for dataA and dataB - cache := CreateCache(sizeA+sizeB, inner).(*cache) - - // Warm the cache with A and B - validateLoad(t, cache, idA, sizeA, dataA, nil) - validateLoad(t, cache, idB, sizeB, dataB, nil) - - // These should have both cache missed, calling into inner.Load() - if 2 != loadCallCount { - t.Errorf("invalid load call count, expected %v got %v", 2, loadCallCount) - } - - // Request load of a third resource, that will overflow the cache - validateLoad(t, cache, idC, sizeC, dataC, nil) - - if 3 != loadCallCount { - t.Errorf("invalid load call count, expected %v got %v", 3, loadCallCount) - } - - // The cache should now be holding B and C, with A evicted. - // Requesting B should not call inner.Load() as it should be cached - validateLoad(t, cache, idB, sizeB, dataB, nil) - if 3 != loadCallCount { - t.Errorf("invalid load call count, expected %v got %v", 3, loadCallCount) - } - - // Requesting C should not call inner.Load() as it should be cached - validateLoad(t, cache, idC, sizeC, dataC, nil) - if 3 != loadCallCount { - t.Errorf("invalid load call count, expected %v got %v", 3, loadCallCount) - } - - // Requesting A should call inner.Load() as it should not be cached - validateLoad(t, cache, idA, sizeA, dataA, nil) - if 4 != loadCallCount { - t.Errorf("invalid load call count, expected %v got %v", 4, loadCallCount) - } -} - -func TestCacheReplace(t *testing.T) { - id := binary.NewID([]byte("123")) - dataA := &Blob{Data: []byte{1, 2, 3}} - dataB := &Blob{Data: []byte{4, 5, 6, 7, 8, 9}} - sizeB := calcSize(dataB) - - inner := createMockStore() - - cache := CreateCache(64, inner).(*cache) - - // Fill the cache with dataA - cache.Store(id, dataA, encode(dataA), nil) - // Replace dataA with dataB - cache.Store(id, dataB, encode(dataB), nil) - - // We expect dataB to be loaded - validateLoad(t, cache, id, sizeB, dataB, nil) -} diff --git a/database/store/store_binary.go b/database/store/store_binary.go index 82e5da2ef..a4e2da01c 100644 --- a/database/store/store_binary.go +++ b/database/store/store_binary.go @@ -12,73 +12,13 @@ import ( ) func init() { - registry.Add((*Blob)(nil).Class()) registry.Add((*keyValue)(nil).Class()) } var ( - binaryIDBlob = binary.ID{0x82, 0x3c, 0x72, 0x55, 0x53, 0x73, 0x2a, 0xab, 0x7c, 0xd0, 0xad, 0x23, 0xd9, 0xf6, 0x5f, 0xd6, 0xf4, 0x54, 0x3c, 0x66} binaryIDkeyValue = binary.ID{0x4f, 0xab, 0x88, 0xad, 0xe2, 0xbc, 0x26, 0x85, 0xfc, 0x31, 0x21, 0xee, 0x4d, 0xcd, 0x67, 0x79, 0x35, 0xeb, 0x4a, 0x9f} ) -type binaryClassBlob struct{} - -func (*Blob) Class() binary.Class { - return (*binaryClassBlob)(nil) -} -func doEncodeBlob(e binary.Encoder, o *Blob) error { - if err := e.Uint32(uint32(len(o.Data))); err != nil { - return err - } - if err := e.Data(o.Data); err != nil { - return err - } - return nil -} -func doDecodeBlob(d binary.Decoder, o *Blob) error { - if count, err := d.Uint32(); err != nil { - return err - } else { - o.Data = make([]byte, count) - if err := d.Data(o.Data); err != nil { - return err - } - } - return nil -} -func doSkipBlob(d binary.Decoder) error { - if count, err := d.Uint32(); err != nil { - return err - } else { - if err := d.Skip(count); err != nil { - return err - } - } - return nil -} -func (*binaryClassBlob) ID() binary.ID { return binaryIDBlob } -func (*binaryClassBlob) New() binary.Object { return &Blob{} } -func (*binaryClassBlob) Encode(e binary.Encoder, obj binary.Object) error { - return doEncodeBlob(e, obj.(*Blob)) -} -func (*binaryClassBlob) Decode(d binary.Decoder) (binary.Object, error) { - obj := &Blob{} - return obj, doDecodeBlob(d, obj) -} -func (*binaryClassBlob) DecodeTo(d binary.Decoder, obj binary.Object) error { - return doDecodeBlob(d, obj.(*Blob)) -} -func (*binaryClassBlob) Skip(d binary.Decoder) error { return doSkipBlob(d) } -func (*binaryClassBlob) Schema() *schema.Class { return schemaBlob } - -var schemaBlob = &schema.Class{ - TypeID: binaryIDBlob, - Name: "Blob", - Fields: []schema.Field{ - {Declared: "Data", Type: &schema.Slice{Alias: "", ValueType: &schema.Primitive{Name: "byte", Method: schema.Uint8}}}, - }, -} - type binaryClasskeyValue struct{} func (*keyValue) Class() binary.Class { -- cgit v1.2.3