martin/databag
1
0
Fork 0
mirror of https://github.com/balzack/databag.git synced 2025-02-12 03:29:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

added second transform queue

Browse Source
This commit is contained in:
Roland Osborne 2022-03-01 10:35:14 -08:00
parent dabfa09eb9
commit ef88a857ed
7 changed files with 115 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/api.oa3
View File

@ -2225,6 +2225,12 @@ paths:
required: true
schema:
type: string
- name: confirm
in: query
description: set if intial state is confirmed
required: false
schema:
type: boolean
responses:
'201':
description: entry created

8
net/server/internal/api_addChannelTopic.go
View File

@ -9,6 +9,8 @@ import (
func AddChannelTopic(w http.ResponseWriter, r *http.Request) {
confirm := r.FormValue("confirm")
var subject Subject
if err := ParseRequest(r, w, &subject); err != nil {
ErrResponse(w, http.StatusBadRequest, err)
@ -33,7 +35,11 @@ func AddChannelTopic(w http.ResponseWriter, r *http.Request) {
topic.Guid = guid
topic.DetailRevision = act.ChannelRevision + 1
topic.TagRevision = act.ChannelRevision + 1
topic.Status = APP_TOPICUNCONFIRMED
if confirm == "true" {
topic.Status = APP_TOPICCONFIRMED
} else {
topic.Status = APP_TOPICUNCONFIRMED
}
topic.Transform = APP_TRANSFORMCOMPLETE
if res := tx.Save(topic).Error; res != nil {
return res

14
net/server/internal/api_addChannelTopicAsset.go
View File

@ -3,6 +3,7 @@ package databag
import (
"os"
"io"
"strings"
"errors"
"github.com/google/uuid"
"net/http"
@ -103,8 +104,17 @@ func AddChannelTopicAsset(w http.ResponseWriter, r *http.Request) {
asset.ChannelID = channelSlot.Channel.ID
asset.TopicID = topicSlot.Topic.ID
asset.Status = APP_ASSETWAITING
asset.Transform = transform
asset.TransformId = id
t := strings.Split(transform, ";")
if len(t) > 0 {
asset.Transform = t[0]
}
if len(t) > 1 {
asset.TransformQueue = t[1]
}
if len(t) > 2 {
asset.TransformParams = t[2]
}
if res := tx.Save(asset).Error; res != nil {
return res
}
@ -135,7 +145,7 @@ func AddChannelTopicAsset(w http.ResponseWriter, r *http.Request) {
}
// invoke transcoder
go transcode()
transcode()
// determine affected contact list
cards := make(map[string]store.Card)

3
net/server/internal/appValues.go
View File

@ -35,6 +35,9 @@ const APP_ASSETPROCESSING = "processing"
const APP_ASSETERROR = "error"
const APP_TRANSFORMCOMPLETE = "complete"
const APP_TRANSFORMINCOMPLETE = "incomplete"
const APP_TRANSFORMERROR = "error"
const APP_TRANSFORMQUEUEA = "A"
const APP_TRANSFORMQUEUEB = "B"
func AppCardStatus(status string) bool {
if status == APP_CARDPENDING {

3
net/server/internal/store/schema.go
View File

@ -245,7 +245,8 @@ type Asset struct {
Crc uint32
Transform string
TransformId string
TransformData string
TransformParams string
TransformQueue string
Created int64 `gorm:"autoCreateTime"`
Updated int64 `gorm:"autoUpdateTime"`
Account Account

133
net/server/internal/transcodeUtil.go
View File

@ -13,73 +13,102 @@ import (
"gorm.io/gorm"
)
var transcodeSync sync.Mutex
var aSync sync.Mutex
var bSync sync.Mutex
func transcode() {
transcodeSync.Lock()
defer transcodeSync.Unlock()
// quick transforms should use A (eg image resize)
go transcodeA()
var assets []store.Asset
if err := store.DB.Preload("Account").Preload("Channel.Cards").Preload("Channel.Groups.Cards").Preload("Channel.ChannelSlot").Preload("Topic.TopicSlot").Where("status = ?", APP_ASSETWAITING).Find(&assets).Error; err != nil {
ErrMsg(err)
return
// slow transofrms should use B (eg video transcode)
go transcodeB()
}
func transcodeA() {
aSync.Lock()
defer aSync.Unlock()
for ;; {
var asset store.Asset
if err := store.DB.Order("created asc").Preload("Account").Preload("Channel.Cards").Preload("Channel.Groups.Cards").Preload("Channel.ChannelSlot").Preload("Topic.TopicSlot").Where("transform_queue = ? AND status = ?", APP_TRANSFORMQUEUEA, APP_ASSETWAITING).First(&asset).Error; err != nil {
if !errors.Is(err, gorm.ErrRecordNotFound) {
ErrMsg(err)
}
return
}
transcodeAsset(&asset)
}
}
func transcodeB() {
bSync.Lock()
defer bSync.Unlock()
for ;; {
var asset store.Asset
if err := store.DB.Order("created asc").Preload("Account").Preload("Channel.Cards").Preload("Channel.Groups.Cards").Preload("Channel.ChannelSlot").Preload("Topic.TopicSlot").Where("transform_queue != ? AND status = ?", APP_TRANSFORMQUEUEB, APP_ASSETWAITING).First(&asset).Error; err != nil {
if !errors.Is(err, gorm.ErrRecordNotFound) {
ErrMsg(err)
}
return
}
transcodeAsset(&asset)
}
}
func transcodeAsset(asset *store.Asset) {
// prepare script path
data := getStrConfigValue(CONFIG_ASSETPATH, ".")
script := getStrConfigValue(CONFIG_SCRIPTPATH, ".")
re := regexp.MustCompile("^[a-zA-Z0-9_]*$")
for _, asset := range assets {
if !re.MatchString(asset.Transform) {
ErrMsg(errors.New("invalid transformi"))
if err := UpdateAsset(&asset, APP_ASSETERROR, 0, 0); err != nil {
if !re.MatchString(asset.Transform) {
ErrMsg(errors.New("invalid transform"))
if err := UpdateAsset(asset, APP_ASSETERROR, 0, 0); err != nil {
ErrMsg(err)
}
} else {
input := data + "/" + asset.TransformId
output := data + "/" + asset.AssetId
cmd := exec.Command(script + "/" + asset.Transform + ".sh", input, output, asset.TransformParams)
var out bytes.Buffer
cmd.Stdout = &out
if err := cmd.Run(); err != nil {
LogMsg(out.String())
ErrMsg(err)
if err := UpdateAsset(asset, APP_ASSETERROR, 0, 0); err != nil {
ErrMsg(err)
}
} else {
input := data + "/" + asset.TransformId
output := data + "/" + asset.AssetId
cmd := exec.Command(script + "/" + asset.Transform + ".sh", input, output)
var out bytes.Buffer
cmd.Stdout = &out
if err := cmd.Run(); err != nil {
LogMsg(out.String())
LogMsg(out.String())
crc, size, err := ScanAsset(output)
if err != nil {
ErrMsg(err)
if err := UpdateAsset(&asset, APP_ASSETERROR, 0, 0); err != nil {
ErrMsg(err)
}
} else {
LogMsg(out.String())
crc, size, err := ScanAsset(output)
if err != nil {
ErrMsg(err)
if err := UpdateAsset(&asset, APP_ASSETERROR, 0, 0); err != nil {
ErrMsg(err)
}
} else if err := UpdateAsset(&asset, APP_ASSETREADY, crc, size); err != nil {
if err := UpdateAsset(asset, APP_ASSETERROR, 0, 0); err != nil {
ErrMsg(err)
}
} else if err := UpdateAsset(asset, APP_ASSETREADY, crc, size); err != nil {
ErrMsg(err)
}
}
}
}
func isComplete(status string, asset *store.Asset) (complete bool, err error) {
if status == APP_ASSETREADY {
var assets []store.Asset
if err = store.DB.Where("topic_id = ?", asset.Topic.ID).Find(&assets).Error; err != nil {
return
}
for _, a := range asset.Topic.Assets {
if a.ID != asset.ID && asset.Status != APP_ASSETREADY {
return
}
}
complete = true
func isComplete(id uint) (complete bool, err error) {
var assets []store.Asset
if err = store.DB.Where("topic_id = ?", id).Find(&assets).Error; err != nil {
return
}
for _, asset := range assets {
if id != asset.ID && asset.Status != APP_ASSETREADY {
return
}
}
complete = true
return
}
@ -93,14 +122,20 @@ func UpdateAsset(asset *store.Asset, status string, crc uint32, size int64) (err
if res := tx.Save(asset).Error; res != nil {
return res
}
complete, ret := isComplete(status, asset)
if ret != nil {
return ret
}
if complete {
if res := tx.Model(&asset.Topic).Update("transform", APP_TRANSFORMCOMPLETE).Error; res != nil {
if status == APP_ASSETERROR {
if res := tx.Model(&asset.Topic).Update("transform", APP_TRANSFORMERROR).Error; res != nil {
return res
}
} else if status == APP_ASSETREADY {
complete, ret := isComplete(asset.ID)
if ret != nil {
return ret
}
if complete {
if res := tx.Model(&asset.Topic).Update("transform", APP_TRANSFORMCOMPLETE).Error; res != nil {
return res
}
}
}
if res := tx.Model(&asset.Topic).Update("detail_revision", act.ChannelRevision + 1).Error; res != nil {
return res

2
net/server/internal/ucTopicShare_test.go
View File

@ -92,7 +92,7 @@ func TestTopicShare(t *testing.T) {
// add asset to topic
assets := &[]Asset{}
params["topicId"] = topic.Id
transforms, err := json.Marshal([]string{ "P01", "P02", "P03" })
transforms, err := json.Marshal([]string{ "P01;A;1234", "P02", "P03" })
assert.NoError(t, err)
assert.NoError(t, ApiTestUpload(AddChannelTopicAsset, "POST", "/content/channels/{channelId}/topics/{topicId}/assets?transforms=" + url.QueryEscape(string(transforms)),
&params, img, APP_TOKENCONTACT, set.C.A.Token, assets, nil))