From 6dd261ca1205a58e68e7c9adf3e608ad6f08f9d9 Mon Sep 17 00:00:00 2001 From: Roland Osborne Date: Mon, 28 Feb 2022 14:59:29 -0800 Subject: [PATCH] testing file upload --- doc/api.oa3 | 4 +- net/server/internal/api_addAccount.go | 13 +- .../internal/api_addChannelTopicAsset.go | 175 ++++++++++++++++++ net/server/internal/api_content.go | 5 - .../internal/api_setChannelTopicConfirmed.go | 11 +- .../internal/api_setChannelTopicSubject.go | 12 +- net/server/internal/api_setProfile.go | 4 +- net/server/internal/appValues.go | 4 + net/server/internal/configUtil.go | 1 + net/server/internal/main_test.go | 11 ++ net/server/internal/routers.go | 4 +- net/server/internal/store/schema.go | 2 +- net/server/internal/testUtil.go | 57 ++++++ net/server/internal/ucTopicShare_test.go | 25 ++- 14 files changed, 303 insertions(+), 25 deletions(-) create mode 100644 net/server/internal/api_addChannelTopicAsset.go diff --git a/doc/api.oa3 b/doc/api.oa3 index acd8b5d5..e0a9e5fe 100644 --- a/doc/api.oa3 +++ b/doc/api.oa3 @@ -2540,7 +2540,9 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Asset' + type: array + items: + $ref: '#/components/schemas/Asset' '401': description: permission denied '404': diff --git a/net/server/internal/api_addAccount.go b/net/server/internal/api_addAccount.go index 11593869..5701ae59 100644 --- a/net/server/internal/api_addAccount.go +++ b/net/server/internal/api_addAccount.go @@ -1,6 +1,7 @@ package databag import ( + "os" "net/http" "crypto/sha256" "encoding/hex" @@ -40,6 +41,12 @@ func AddAccount(w http.ResponseWriter, r *http.Request) { hash := sha256.Sum256(msg) fingerprint := hex.EncodeToString(hash[:]) + // create path for account data + path := getStrConfigValue(CONFIG_ASSETPATH, ".") + "/" + fingerprint + if err := os.Mkdir(path, os.ModePerm); err != nil { + ErrResponse(w, http.StatusInternalServerError, err) + } + // create new account account := store.Account{ Username: username, @@ -54,14 +61,14 @@ func AddAccount(w http.ResponseWriter, r *http.Request) { // save account and delete token err = store.DB.Transaction(func(tx *gorm.DB) error { - if res := store.DB.Create(&detail).Error; res != nil { + if res := tx.Create(&detail).Error; res != nil { return res; } account.AccountDetailID = detail.ID - if res := store.DB.Create(&account).Error; res != nil { + if res := tx.Create(&account).Error; res != nil { return res; } - if res := store.DB.Delete(token).Error; res != nil { + if res := tx.Delete(token).Error; res != nil { return res; } return nil; diff --git a/net/server/internal/api_addChannelTopicAsset.go b/net/server/internal/api_addChannelTopicAsset.go new file mode 100644 index 00000000..0d264eb0 --- /dev/null +++ b/net/server/internal/api_addChannelTopicAsset.go @@ -0,0 +1,175 @@ +package databag + +import ( + "os" + "io" + "errors" + "github.com/google/uuid" + "net/http" + "hash/crc32" + "github.com/gorilla/mux" + "gorm.io/gorm" + "databag/internal/store" + "encoding/json" +) + +func AddChannelTopicAsset(w http.ResponseWriter, r *http.Request) { + + // scan parameters + params := mux.Vars(r) + topicId := params["topicId"] + var transforms []string + if r.FormValue("transforms") != "" { + if err := json.Unmarshal([]byte(r.FormValue("transforms")), &transforms); err != nil { + ErrResponse(w, http.StatusBadRequest, errors.New("invalid asset transform")) + return + } + } + +PrintMsg(transforms) + + channelSlot, guid, err, code := getChannelSlot(r, true) + if err != nil { + ErrResponse(w, code, err) + return + } + act := &channelSlot.Account + + // load topic + var topicSlot store.TopicSlot + if err = store.DB.Preload("Topic").Where("channel_id = ? AND topic_slot_id = ?", channelSlot.Channel.ID, topicId).First(&topicSlot).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + ErrResponse(w, http.StatusNotFound, err) + } else { + ErrResponse(w, http.StatusInternalServerError, err) + } + return + } + if topicSlot.Topic == nil { + ErrResponse(w, http.StatusNotFound, errors.New("referenced empty topic")) + return + } + + // can only update topic if creator + if topicSlot.Topic.Guid != guid { + ErrResponse(w, http.StatusUnauthorized, errors.New("topic not created by you")) + return + } + + // save new file + id := uuid.New().String() + path := getStrConfigValue(CONFIG_ASSETPATH, ".") + "/" + channelSlot.Account.Guid + "/" + id + if err := r.ParseMultipartForm(32 << 20); err != nil { + ErrResponse(w, http.StatusBadRequest, err) + return + } + file, _, err := r.FormFile("asset") + if err != nil { + ErrResponse(w, http.StatusBadRequest, err) + return + } + defer file.Close() + crc, size, err := SaveAsset(file, path) + if err != nil { + ErrResponse(w, http.StatusInternalServerError, err) + return + } + + assets := []Asset{} + asset := &store.Asset{} + asset.AssetId = id + asset.AccountID = channelSlot.Account.ID + asset.TopicID = topicSlot.Topic.ID + asset.Status = APP_ASSETREADY + asset.Size = size + asset.Crc = crc + err = store.DB.Transaction(func(tx *gorm.DB) error { + if res := tx.Save(asset).Error; res != nil { + return res + } + assets = append(assets, Asset{ AssetId: id, Status: APP_ASSETREADY}) + for _, transform := range transforms { + asset := &store.Asset{} + asset.AssetId = uuid.New().String() + asset.AccountID = channelSlot.Account.ID + asset.TopicID = topicSlot.Topic.ID + asset.Status = APP_ASSETWAITING + asset.Transform = transform + asset.TransformId = id + if res := tx.Save(asset).Error; res != nil { + return res + } + assets = append(assets, Asset{ AssetId: asset.AssetId, Transform: transform, Status: APP_ASSETWAITING}) + } + if res := tx.Model(&topicSlot).Update("revision", act.ChannelRevision + 1).Error; res != nil { + return res + } + if res := tx.Model(&channelSlot).Update("revision", act.ChannelRevision + 1).Error; res != nil { + return res + } + if res := tx.Model(act).Update("channel_revision", act.ChannelRevision + 1).Error; res != nil { + return res + } + return nil + }) + if err != nil { + ErrResponse(w, http.StatusInternalServerError, err) + return + } + + // determine affected contact list + cards := make(map[string]store.Card) + for _, card := range channelSlot.Channel.Cards { + cards[card.Guid] = card + } + for _, group := range channelSlot.Channel.Groups { + for _, card := range group.Cards { + cards[card.Guid] = card + } + } + + SetStatus(act) + for _, card := range cards { + SetContactChannelNotification(act, &card) + } + WriteResponse(w, &assets) +} + +func SaveAsset(src io.Reader, path string) (crc uint32, size int64, err error) { + + output, res := os.OpenFile(path, os.O_WRONLY | os.O_CREATE, 0666) + if res != nil { + err = res + return + } + defer output.Close() + + // prepare hash + table := crc32.MakeTable(crc32.IEEE) + + // compute has as data is saved + data := make([]byte, 4096) + for { + n, res := src.Read(data) + if res != nil { + if res == io.EOF { + break + } + err = res + return + } + + crc = crc32.Update(crc, table, data[:n]) + output.Write(data[:n]) + } + + // read size + info, ret := output.Stat() + if ret != nil { + err = ret + return + } + size = info.Size() + return +} + diff --git a/net/server/internal/api_content.go b/net/server/internal/api_content.go index 4d3d7c2f..24858780 100644 --- a/net/server/internal/api_content.go +++ b/net/server/internal/api_content.go @@ -13,11 +13,6 @@ import ( "net/http" ) -func AddChannelAsset(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json; charset=UTF-8") - w.WriteHeader(http.StatusOK) -} - func AddChannelTopicTag(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json; charset=UTF-8") w.WriteHeader(http.StatusOK) diff --git a/net/server/internal/api_setChannelTopicConfirmed.go b/net/server/internal/api_setChannelTopicConfirmed.go index 7bca1555..1e4222fe 100644 --- a/net/server/internal/api_setChannelTopicConfirmed.go +++ b/net/server/internal/api_setChannelTopicConfirmed.go @@ -33,17 +33,20 @@ func SetChannelTopicConfirmed(w http.ResponseWriter, r *http.Request) { // load topic var topicSlot store.TopicSlot - if err = store.DB.Where("channel_id = ? AND topic_slot_id = ?", channelSlot.Channel.ID, topicId).First(&topicSlot).Error; err != nil { + if err = store.DB.Preload("Topic").Where("channel_id = ? AND topic_slot_id = ?", channelSlot.Channel.ID, topicId).First(&topicSlot).Error; err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { - code = http.StatusNotFound + ErrResponse(w, http.StatusNotFound, err) } else { - code = http.StatusInternalServerError + ErrResponse(w, http.StatusInternalServerError, err) } return } + if topicSlot.Topic == nil { + ErrResponse(w, http.StatusNotFound, errors.New("referenced empty slot")) + return + } err = store.DB.Transaction(func(tx *gorm.DB) error { - if res := tx.Model(topicSlot.Topic).Update("status", status).Error; res != nil { return res } diff --git a/net/server/internal/api_setChannelTopicSubject.go b/net/server/internal/api_setChannelTopicSubject.go index a4dcca92..421f18b3 100644 --- a/net/server/internal/api_setChannelTopicSubject.go +++ b/net/server/internal/api_setChannelTopicSubject.go @@ -20,7 +20,7 @@ func SetChannelTopicSubject(w http.ResponseWriter, r *http.Request) { return } - channelSlot, _, err, code := getChannelSlot(r, true) + channelSlot, guid, err, code := getChannelSlot(r, true) if err != nil { ErrResponse(w, code, err) return @@ -31,13 +31,19 @@ func SetChannelTopicSubject(w http.ResponseWriter, r *http.Request) { var topicSlot store.TopicSlot if err = store.DB.Where("channel_id = ? AND topic_slot_id = ?", channelSlot.Channel.ID, topicId).First(&topicSlot).Error; err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { - code = http.StatusNotFound + ErrResponse(w, http.StatusNotFound, err) } else { - code = http.StatusInternalServerError + ErrResponse(w, http.StatusInternalServerError, err) } return } + // can only update subject if creator + if topicSlot.Topic.Guid != guid { + ErrResponse(w, http.StatusUnauthorized, errors.New("topic not created by you")) + return + } + err = store.DB.Transaction(func(tx *gorm.DB) error { if res := tx.Model(topicSlot.Topic).Update("data", subject.Data).Error; res != nil { diff --git a/net/server/internal/api_setProfile.go b/net/server/internal/api_setProfile.go index 3315ea9e..3ff8b8f0 100644 --- a/net/server/internal/api_setProfile.go +++ b/net/server/internal/api_setProfile.go @@ -28,10 +28,10 @@ func SetProfile(w http.ResponseWriter, r *http.Request) { account.AccountDetail.Description = profileData.Description err = store.DB.Transaction(func(tx *gorm.DB) error { - if res := store.DB.Save(&account.AccountDetail).Error; res != nil { + if res := tx.Save(&account.AccountDetail).Error; res != nil { return res } - if res := store.DB.Model(&account).Update("profile_revision", account.ProfileRevision + 1).Error; res != nil { + if res := tx.Model(&account).Update("profile_revision", account.ProfileRevision + 1).Error; res != nil { return res } return nil diff --git a/net/server/internal/appValues.go b/net/server/internal/appValues.go index fd9003f3..7e210a5b 100644 --- a/net/server/internal/appValues.go +++ b/net/server/internal/appValues.go @@ -29,6 +29,10 @@ const APP_TOKENCONTACT = "contact" const APP_NOTIFYBUFFER = 4096 const APP_TOPICUNCONFIRMED = "unconfirmed" const APP_TOPICCONFIRMED = "confirmed" +const APP_ASSETREADY = "ready" +const APP_ASSETWAITING = "waiting" +const APP_ASSETPROCESSING = "processing" +const APP_ASSETERROR = "error" func AppCardStatus(status string) bool { if status == APP_CARDPENDING { diff --git a/net/server/internal/configUtil.go b/net/server/internal/configUtil.go index b1134ce8..ad1e1784 100644 --- a/net/server/internal/configUtil.go +++ b/net/server/internal/configUtil.go @@ -12,6 +12,7 @@ const CONFIG_PASSWORD = "password" const CONFIG_DOMAIN = "domain" const CONFIG_PUBLICLIMIT = "public_limit" const CONFIG_STORAGE = "storage" +const CONFIG_ASSETPATH = "asset_path" func getStrConfigValue(configId string, empty string) string { var config store.Config diff --git a/net/server/internal/main_test.go b/net/server/internal/main_test.go index 29300c62..d4bc21ca 100644 --- a/net/server/internal/main_test.go +++ b/net/server/internal/main_test.go @@ -11,7 +11,12 @@ func TestMain(m *testing.M) { // SetHideLog(true) SetKeySize(2048) os.Remove("databag.db") + os.RemoveAll("testdata") + store.SetPath("databag.db") + if err := os.Mkdir("testdata", os.ModePerm); err != nil { + panic("failed to create testdata path") + } r, w, _ := NewRequest("GET", "/admin/status", nil) GetNodeStatus(w, r) @@ -28,6 +33,12 @@ func TestMain(m *testing.M) { panic("failed to claim server") } + // config data path + path := &store.Config{ ConfigId: CONFIG_ASSETPATH, StrValue: "./testdata" } + if err := store.DB.Save(path).Error; err != nil { + panic("failed to configure datapath") + } + // config server config := NodeConfig{Domain: "example.com", PublicLimit: 1024, AccountStorage: 4096} r, w, _ = NewRequest("PUT", "/admin/config", &config) diff --git a/net/server/internal/routers.go b/net/server/internal/routers.go index a559b4a4..e76658f5 100644 --- a/net/server/internal/routers.go +++ b/net/server/internal/routers.go @@ -496,10 +496,10 @@ var routes = Routes{ }, Route{ - "AddChannelAsset", + "AddChannelTopicAsset", strings.ToUpper("Post"), "/content/channels/{channelId}/topics/{topicId}/assets", - AddChannelAsset, + AddChannelTopicAsset, }, Route{ diff --git a/net/server/internal/store/schema.go b/net/server/internal/store/schema.go index 3804f94d..c78d21d6 100644 --- a/net/server/internal/store/schema.go +++ b/net/server/internal/store/schema.go @@ -238,7 +238,7 @@ type Asset struct { AccountID uint `gorm:"not null;index:asset,unique"` TopicID uint Status string `gorm:"not null;index"` - Size uint64 + Size int64 Crc uint32 Transform string TransformId string diff --git a/net/server/internal/testUtil.go b/net/server/internal/testUtil.go index 55acf9e7..469e6e55 100644 --- a/net/server/internal/testUtil.go +++ b/net/server/internal/testUtil.go @@ -3,6 +3,8 @@ package databag import ( "io/ioutil" "errors" + "bytes" + "mime/multipart" "strings" "strconv" "time" @@ -133,6 +135,61 @@ func ApiTestMsg( return } +func ApiTestUpload( + endpoint func(http.ResponseWriter, *http.Request), + requestType string, + name string, + params *map[string]string, + body []byte, + tokenType string, + token string, + response interface{}, + responseHeader *map[string][]string, + ) (err error) { + + data := bytes.Buffer{} + writer := multipart.NewWriter(&data) + part, err := writer.CreateFormFile("asset", "asset") + if err != nil { + return err + } + part.Write(body) + if err = writer.Close(); err != nil { + return + } + + w := httptest.NewRecorder() + r := httptest.NewRequest(requestType, name, &data) + + if params != nil { + r = mux.SetURLVars(r, *params) + } + if tokenType != "" { + r.Header.Add("TokenType", tokenType) + } + if token != "" { + SetBearerAuth(r, token) + } + r.Header.Set("Content-Type", writer.FormDataContentType()) + endpoint(w, r) + + resp := w.Result() + if resp.StatusCode != 200 { + err = errors.New("response failed"); + return + } + if responseHeader != nil { + *responseHeader = resp.Header + } + if response == nil { + return + } + dec := json.NewDecoder(resp.Body) + dec.Decode(response) + return +} + + // // A --- connected,group connected,group --- B // | \ /| diff --git a/net/server/internal/ucTopicShare_test.go b/net/server/internal/ucTopicShare_test.go index 9e5bb9da..231f2106 100644 --- a/net/server/internal/ucTopicShare_test.go +++ b/net/server/internal/ucTopicShare_test.go @@ -5,6 +5,8 @@ import ( "testing" "encoding/base64" "github.com/stretchr/testify/assert" + "encoding/json" + "net/url" ) func TestTopicShare(t *testing.T) { @@ -78,13 +80,28 @@ func TestTopicShare(t *testing.T) { subject = &Subject{ DataType: "topicdatatype", Data: "subjectfromB" } assert.NoError(t, ApiTestMsg(AddChannelTopic, "POST", "/content/channels/{channelId}/topics", ¶ms, subject, APP_TOKENCONTACT, set.B.A.Token, topic, nil)) - topic = &Topic{} + params["topicId"] = topic.Id + assert.NoError(t, ApiTestMsg(SetChannelTopicConfirmed, "PUT", "/content/channels/{channelId}/topics/{topicId}/confirmed", + ¶ms, APP_TOPICCONFIRMED, APP_TOKENCONTACT, set.B.A.Token, nil, nil)) + topic = &Topic{} subject = &Subject{ DataType: "topicdatatype", Data: "subjectfromC" } assert.NoError(t, ApiTestMsg(AddChannelTopic, "POST", "/content/channels/{channelId}/topics", ¶ms, subject, APP_TOKENCONTACT, set.C.A.Token, topic, nil)) - - PrintMsg(topic) + // add asset to topic + assets := &[]Asset{} + params["topicId"] = topic.Id + transforms, err := json.Marshal([]string{ "P01", "P02", "P03" }) + assert.NoError(t, err) + assert.NoError(t, ApiTestUpload(AddChannelTopicAsset, "POST", "/content/channels/{channelId}/topics/{topicId}/assets?transforms=" + url.QueryEscape(string(transforms)), + ¶ms, img, APP_TOKENCONTACT, set.C.A.Token, assets, nil)) + PrintMsg(assets) +PrintMsg(len(img)) + + // view topics + topics := &[]Topic{} + assert.NoError(t, ApiTestMsg(GetChannelTopics, "GET", "/content/channels/{channelId}/topics", + ¶ms, nil, APP_TOKENAPP, set.A.Token, topics, nil)) + } -