d77991c95a
* feat: port * fix: use httprouter * fix: WriteHeader * fix: bolthold * fix: bugs * chore: one less file * test: test handler * fix: bug in id * test: fix cases * chore: tidy * fix: use atomic.Int32 * fix: use atomic.Store * feat: support close * chore: lint * fix: cache keys are case insensitive * fix: options * fix: use options * fix: close * fix: ignore close error * Revert "fix: close" This reverts commit d53ea7568ba03908eb153031c435008fd47e7ccb. * fix: cacheUrlKey * fix: nil close * chore: lint code * fix: test key * test: case insensitive * chore: lint
469 lines
14 KiB
Go
469 lines
14 KiB
Go
package artifactcache
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/rand"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"go.etcd.io/bbolt"
|
|
)
|
|
|
|
func TestHandler(t *testing.T) {
|
|
dir := filepath.Join(t.TempDir(), "artifactcache")
|
|
handler, err := StartHandler(dir, "", 0, nil)
|
|
require.NoError(t, err)
|
|
|
|
base := fmt.Sprintf("%s%s", handler.ExternalURL(), urlBase)
|
|
|
|
defer func() {
|
|
t.Run("inpect db", func(t *testing.T) {
|
|
require.NoError(t, handler.db.Bolt().View(func(tx *bbolt.Tx) error {
|
|
return tx.Bucket([]byte("Cache")).ForEach(func(k, v []byte) error {
|
|
t.Logf("%s: %s", k, v)
|
|
return nil
|
|
})
|
|
}))
|
|
})
|
|
t.Run("close", func(t *testing.T) {
|
|
require.NoError(t, handler.Close())
|
|
assert.Nil(t, handler.server)
|
|
assert.Nil(t, handler.listener)
|
|
assert.Nil(t, handler.db)
|
|
_, err := http.Post(fmt.Sprintf("%s/caches/%d", base, 1), "", nil)
|
|
assert.Error(t, err)
|
|
})
|
|
}()
|
|
|
|
t.Run("get not exist", func(t *testing.T) {
|
|
key := strings.ToLower(t.Name())
|
|
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
|
resp, err := http.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version))
|
|
require.NoError(t, err)
|
|
require.Equal(t, 204, resp.StatusCode)
|
|
})
|
|
|
|
t.Run("reserve and upload", func(t *testing.T) {
|
|
key := strings.ToLower(t.Name())
|
|
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
|
content := make([]byte, 100)
|
|
_, err := rand.Read(content)
|
|
require.NoError(t, err)
|
|
uploadCacheNormally(t, base, key, version, content)
|
|
})
|
|
|
|
t.Run("clean", func(t *testing.T) {
|
|
resp, err := http.Post(fmt.Sprintf("%s/clean", base), "", nil)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
})
|
|
|
|
t.Run("reserve with bad request", func(t *testing.T) {
|
|
body := []byte(`invalid json`)
|
|
require.NoError(t, err)
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body))
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 400, resp.StatusCode)
|
|
})
|
|
|
|
t.Run("duplicate reserve", func(t *testing.T) {
|
|
key := strings.ToLower(t.Name())
|
|
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
|
{
|
|
body, err := json.Marshal(&Request{
|
|
Key: key,
|
|
Version: version,
|
|
Size: 100,
|
|
})
|
|
require.NoError(t, err)
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body))
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
|
|
got := struct {
|
|
CacheID uint64 `json:"cacheId"`
|
|
}{}
|
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
|
}
|
|
{
|
|
body, err := json.Marshal(&Request{
|
|
Key: key,
|
|
Version: version,
|
|
Size: 100,
|
|
})
|
|
require.NoError(t, err)
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body))
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 400, resp.StatusCode)
|
|
}
|
|
})
|
|
|
|
t.Run("upload with bad id", func(t *testing.T) {
|
|
req, err := http.NewRequest(http.MethodPatch,
|
|
fmt.Sprintf("%s/caches/invalid_id", base), bytes.NewReader(nil))
|
|
require.NoError(t, err)
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
req.Header.Set("Content-Range", "bytes 0-99/*")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 400, resp.StatusCode)
|
|
})
|
|
|
|
t.Run("upload without reserve", func(t *testing.T) {
|
|
req, err := http.NewRequest(http.MethodPatch,
|
|
fmt.Sprintf("%s/caches/%d", base, 1000), bytes.NewReader(nil))
|
|
require.NoError(t, err)
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
req.Header.Set("Content-Range", "bytes 0-99/*")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 400, resp.StatusCode)
|
|
})
|
|
|
|
t.Run("upload with complete", func(t *testing.T) {
|
|
key := strings.ToLower(t.Name())
|
|
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
|
var id uint64
|
|
content := make([]byte, 100)
|
|
_, err := rand.Read(content)
|
|
require.NoError(t, err)
|
|
{
|
|
body, err := json.Marshal(&Request{
|
|
Key: key,
|
|
Version: version,
|
|
Size: 100,
|
|
})
|
|
require.NoError(t, err)
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body))
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
|
|
got := struct {
|
|
CacheID uint64 `json:"cacheId"`
|
|
}{}
|
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
|
id = got.CacheID
|
|
}
|
|
{
|
|
req, err := http.NewRequest(http.MethodPatch,
|
|
fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content))
|
|
require.NoError(t, err)
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
req.Header.Set("Content-Range", "bytes 0-99/*")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
}
|
|
{
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
}
|
|
{
|
|
req, err := http.NewRequest(http.MethodPatch,
|
|
fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content))
|
|
require.NoError(t, err)
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
req.Header.Set("Content-Range", "bytes 0-99/*")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 400, resp.StatusCode)
|
|
}
|
|
})
|
|
|
|
t.Run("upload with invalid range", func(t *testing.T) {
|
|
key := strings.ToLower(t.Name())
|
|
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
|
var id uint64
|
|
content := make([]byte, 100)
|
|
_, err := rand.Read(content)
|
|
require.NoError(t, err)
|
|
{
|
|
body, err := json.Marshal(&Request{
|
|
Key: key,
|
|
Version: version,
|
|
Size: 100,
|
|
})
|
|
require.NoError(t, err)
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body))
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
|
|
got := struct {
|
|
CacheID uint64 `json:"cacheId"`
|
|
}{}
|
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
|
id = got.CacheID
|
|
}
|
|
{
|
|
req, err := http.NewRequest(http.MethodPatch,
|
|
fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content))
|
|
require.NoError(t, err)
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
req.Header.Set("Content-Range", "bytes xx-99/*")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 400, resp.StatusCode)
|
|
}
|
|
})
|
|
|
|
t.Run("commit with bad id", func(t *testing.T) {
|
|
{
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches/invalid_id", base), "", nil)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 400, resp.StatusCode)
|
|
}
|
|
})
|
|
|
|
t.Run("commit with not exist id", func(t *testing.T) {
|
|
{
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches/%d", base, 100), "", nil)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 400, resp.StatusCode)
|
|
}
|
|
})
|
|
|
|
t.Run("duplicate commit", func(t *testing.T) {
|
|
key := strings.ToLower(t.Name())
|
|
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
|
var id uint64
|
|
content := make([]byte, 100)
|
|
_, err := rand.Read(content)
|
|
require.NoError(t, err)
|
|
{
|
|
body, err := json.Marshal(&Request{
|
|
Key: key,
|
|
Version: version,
|
|
Size: 100,
|
|
})
|
|
require.NoError(t, err)
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body))
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
|
|
got := struct {
|
|
CacheID uint64 `json:"cacheId"`
|
|
}{}
|
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
|
id = got.CacheID
|
|
}
|
|
{
|
|
req, err := http.NewRequest(http.MethodPatch,
|
|
fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content))
|
|
require.NoError(t, err)
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
req.Header.Set("Content-Range", "bytes 0-99/*")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
}
|
|
{
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
}
|
|
{
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 400, resp.StatusCode)
|
|
}
|
|
})
|
|
|
|
t.Run("commit early", func(t *testing.T) {
|
|
key := strings.ToLower(t.Name())
|
|
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
|
var id uint64
|
|
content := make([]byte, 100)
|
|
_, err := rand.Read(content)
|
|
require.NoError(t, err)
|
|
{
|
|
body, err := json.Marshal(&Request{
|
|
Key: key,
|
|
Version: version,
|
|
Size: 100,
|
|
})
|
|
require.NoError(t, err)
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body))
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
|
|
got := struct {
|
|
CacheID uint64 `json:"cacheId"`
|
|
}{}
|
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
|
id = got.CacheID
|
|
}
|
|
{
|
|
req, err := http.NewRequest(http.MethodPatch,
|
|
fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content[:50]))
|
|
require.NoError(t, err)
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
req.Header.Set("Content-Range", "bytes 0-59/*")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
}
|
|
{
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 500, resp.StatusCode)
|
|
}
|
|
})
|
|
|
|
t.Run("get with bad id", func(t *testing.T) {
|
|
resp, err := http.Get(fmt.Sprintf("%s/artifacts/invalid_id", base))
|
|
require.NoError(t, err)
|
|
require.Equal(t, 400, resp.StatusCode)
|
|
})
|
|
|
|
t.Run("get with not exist id", func(t *testing.T) {
|
|
resp, err := http.Get(fmt.Sprintf("%s/artifacts/%d", base, 100))
|
|
require.NoError(t, err)
|
|
require.Equal(t, 404, resp.StatusCode)
|
|
})
|
|
|
|
t.Run("get with not exist id", func(t *testing.T) {
|
|
resp, err := http.Get(fmt.Sprintf("%s/artifacts/%d", base, 100))
|
|
require.NoError(t, err)
|
|
require.Equal(t, 404, resp.StatusCode)
|
|
})
|
|
|
|
t.Run("get with multiple keys", func(t *testing.T) {
|
|
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
|
key := strings.ToLower(t.Name())
|
|
keys := [3]string{
|
|
key + "_a",
|
|
key + "_a_b",
|
|
key + "_a_b_c",
|
|
}
|
|
contents := [3][]byte{
|
|
make([]byte, 100),
|
|
make([]byte, 200),
|
|
make([]byte, 300),
|
|
}
|
|
for i := range contents {
|
|
_, err := rand.Read(contents[i])
|
|
require.NoError(t, err)
|
|
uploadCacheNormally(t, base, keys[i], version, contents[i])
|
|
}
|
|
|
|
reqKeys := strings.Join([]string{
|
|
key + "_a_b_x",
|
|
key + "_a_b",
|
|
key + "_a",
|
|
}, ",")
|
|
var archiveLocation string
|
|
{
|
|
resp, err := http.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, reqKeys, version))
|
|
require.NoError(t, err)
|
|
require.Equal(t, 200, resp.StatusCode)
|
|
got := struct {
|
|
Result string `json:"result"`
|
|
ArchiveLocation string `json:"archiveLocation"`
|
|
CacheKey string `json:"cacheKey"`
|
|
}{}
|
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
|
assert.Equal(t, "hit", got.Result)
|
|
assert.Equal(t, keys[1], got.CacheKey)
|
|
archiveLocation = got.ArchiveLocation
|
|
}
|
|
{
|
|
resp, err := http.Get(archiveLocation) //nolint:gosec
|
|
require.NoError(t, err)
|
|
require.Equal(t, 200, resp.StatusCode)
|
|
got, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, contents[1], got)
|
|
}
|
|
})
|
|
|
|
t.Run("case insensitive", func(t *testing.T) {
|
|
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
|
key := strings.ToLower(t.Name())
|
|
content := make([]byte, 100)
|
|
_, err := rand.Read(content)
|
|
require.NoError(t, err)
|
|
uploadCacheNormally(t, base, key+"_ABC", version, content)
|
|
|
|
{
|
|
reqKey := key + "_aBc"
|
|
resp, err := http.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, reqKey, version))
|
|
require.NoError(t, err)
|
|
require.Equal(t, 200, resp.StatusCode)
|
|
got := struct {
|
|
Result string `json:"result"`
|
|
ArchiveLocation string `json:"archiveLocation"`
|
|
CacheKey string `json:"cacheKey"`
|
|
}{}
|
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
|
assert.Equal(t, "hit", got.Result)
|
|
assert.Equal(t, key+"_abc", got.CacheKey)
|
|
}
|
|
})
|
|
}
|
|
|
|
func uploadCacheNormally(t *testing.T, base, key, version string, content []byte) {
|
|
var id uint64
|
|
{
|
|
body, err := json.Marshal(&Request{
|
|
Key: key,
|
|
Version: version,
|
|
Size: int64(len(content)),
|
|
})
|
|
require.NoError(t, err)
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body))
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
|
|
got := struct {
|
|
CacheID uint64 `json:"cacheId"`
|
|
}{}
|
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
|
id = got.CacheID
|
|
}
|
|
{
|
|
req, err := http.NewRequest(http.MethodPatch,
|
|
fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content))
|
|
require.NoError(t, err)
|
|
req.Header.Set("Content-Type", "application/octet-stream")
|
|
req.Header.Set("Content-Range", "bytes 0-99/*")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
}
|
|
{
|
|
resp, err := http.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 200, resp.StatusCode)
|
|
}
|
|
var archiveLocation string
|
|
{
|
|
resp, err := http.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version))
|
|
require.NoError(t, err)
|
|
require.Equal(t, 200, resp.StatusCode)
|
|
got := struct {
|
|
Result string `json:"result"`
|
|
ArchiveLocation string `json:"archiveLocation"`
|
|
CacheKey string `json:"cacheKey"`
|
|
}{}
|
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
|
assert.Equal(t, "hit", got.Result)
|
|
assert.Equal(t, strings.ToLower(key), got.CacheKey)
|
|
archiveLocation = got.ArchiveLocation
|
|
}
|
|
{
|
|
resp, err := http.Get(archiveLocation) //nolint:gosec
|
|
require.NoError(t, err)
|
|
require.Equal(t, 200, resp.StatusCode)
|
|
got, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, content, got)
|
|
}
|
|
}
|