ultrasn0w/huso
1
0
Fork 0
mirror of https://github.com/ultrasn0w/huso.git synced 2025-12-14 10:39:53 +01:00
Code Issues Projects Releases Wiki Activity

Add ring log + 429

Browse Source
This commit is contained in:
daru
2022-05-13 22:33:59 +02:00
parent b3c152e7b1
commit bbf99fadb4
9 changed files with 163 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@@ -15,7 +15,7 @@ require (
require (
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/bwmarrin/snowflake v0.3.0 // indirect
github.com/klauspost/compress v1.15.3 // indirect
github.com/klauspost/compress v1.15.4 // indirect
github.com/savsgio/gotils v0.0.0-20220401102855-e56b59f40436 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect

2
go.sum
View File

@@ -20,6 +20,8 @@ github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.3 h1:wmfu2iqj9q22SyMINp1uQ8C2/V4M1phJdmH9fG4nba0=
github.com/klauspost/compress v1.15.3/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.15.4 h1:1kn4/7MepF/CHmYub99/nNX8az0IJjfSOU/jbnTVfqQ=
github.com/klauspost/compress v1.15.4/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/savsgio/gotils v0.0.0-20220401102855-e56b59f40436 h1:sfTahD3f2BSjx9U3R4K09PkNuZZWthT7g6vzTIXNWkM=

9
huso.go
View File

@@ -2,6 +2,7 @@ package main
import (
"flag"
"fmt"
"log"
"os"
"os/signal"
@@ -46,6 +47,7 @@ var (
animeListCache *bigcache.BigCache
db *nutsdb.DB
jikanLimiter *rate.Limiter
logOut *RingBuf
)
func main() {
@@ -55,9 +57,12 @@ func main() {
sc := make(chan os.Signal, 1)
signal.Notify(sc, syscall.SIGINT, syscall.SIGTERM, os.Interrupt)
color.Notice.Printf("huso %s %s\n", husoVersion, runtime.Version())
logOut = NewRingBuf(1000)
jikanLimiter = rate.NewLimiter(rate.Every(time.Second), 2)
color.Notice.Printf("huso %s %s\n", husoVersion, runtime.Version())
logOut.WriteLine(fmt.Sprintf("huso %s %s", husoVersion, runtime.Version()))
jikanLimiter = rate.NewLimiter(rate.Every(time.Second), 1)
// cache init
var err error

5
knecht.go
View File

@@ -118,11 +118,13 @@ func GetUserData(username string) (*User, []byte, error) {
func GetSeasonDataAll() ([]Anime, []byte, error) {
color.Infoln("Aktuelle Season abfragen...")
logOut.WriteLine("Aktuelle Season abfragen...")
data, _, err := GetSeasonData(1)
if err != nil {
return nil, nil, err
}
color.Infof("%d Anime auf %d Seiten\n", data.Pagination.Items.Total, data.Pagination.LastVisiblePage)
logOut.WriteLine(fmt.Sprintf("%d Anime auf %d Seiten", data.Pagination.Items.Total, data.Pagination.LastVisiblePage))
animes := make([]Anime, 0)
// convert to anime
for _, a := range data.Data {
@@ -130,7 +132,7 @@ func GetSeasonDataAll() ([]Anime, []byte, error) {
}
for i := 2; data.Pagination.HasNextPage; i++ {
color.Infof("Seite %d abfragen...\n", i)
time.Sleep(time.Second)
logOut.WriteLine(fmt.Sprintf("Seite %d abfragen...", i))
newData, _, err := GetSeasonData(i)
if err != nil {
return nil, nil, err
@@ -144,6 +146,7 @@ func GetSeasonDataAll() ([]Anime, []byte, error) {
}
}
color.Infof("%d Anime bekommen\n", len(animes))
logOut.WriteLine(fmt.Sprintf("%d Anime bekommen", len(animes)))
bytes, err := json.Marshal(animes)
return animes, bytes, err
}

7
ober.go
View File

@@ -19,6 +19,7 @@ func RunWebserv() {
r.GET("/api/auth/{user}", Headers(AuthTest))
r.GET("/api/anime/{id}", Headers(AnimeGet))
r.GET("/api/animesearch", Headers(AnimeSearchGet))
r.GET("/api/appointment", Headers(AppointmentGet))
r.GET("/api/user/{user?}", Headers(UserGet))
r.GET("/api/watch/{user?}", Headers(WatchGet))
r.GET("/api/watchext/{user?}", Headers(WatchExtendedGet))
@@ -36,7 +37,7 @@ func Start(ctx *fasthttp.RequestCtx) {
return
}
WriteIndex(ctx, season)
WriteIndex(ctx, season, logOut.String())
ctx.SetContentType("text/html; charset=utf-8")
ctx.SetStatusCode(fasthttp.StatusOK)
@@ -135,6 +136,10 @@ func AnimeSearchGet(ctx *fasthttp.RequestCtx) {
ctx.SetStatusCode(fasthttp.StatusOK)
}
func AppointmentGet(ctx *fasthttp.RequestCtx) {
ctx.SetStatusCode(fasthttp.StatusNotImplemented)
}
func UserGet(ctx *fasthttp.RequestCtx) {
usrVal := ctx.UserValue("user")
users := make([]User, 0)

14
praktikant.go
View File

@@ -2,6 +2,7 @@ package main
import (
"encoding/json"
"fmt"
"strconv"
"time"
@@ -20,10 +21,12 @@ func Arbeit() {
_, bytes, err := GetSeasonDataAll()
if err != nil {
color.Errorln(err.Error())
logOut.WriteLine(err.Error())
} else {
err = seasoncache.Set(seasonApiJikan, bytes)
if err != nil {
color.Errorln(err.Error())
logOut.WriteLine(err.Error())
}
}
// refresh animelist of users
@@ -31,6 +34,7 @@ func Arbeit() {
if err != nil {
if err != nutsdb.ErrBucketEmpty {
color.Errorln(err.Error())
logOut.WriteLine(err.Error())
}
return
}
@@ -41,6 +45,7 @@ func Arbeit() {
newProgress, updated, err := FetchProgress(a.Anime, u.MalID, u.Username, u.Progress)
if err != nil {
color.Errorln(err.Error())
logOut.WriteLine(err.Error())
continue
}
if newProgress == u.Progress {
@@ -48,9 +53,11 @@ func Arbeit() {
}
// update db
color.Infof("%s progress von %d: %d -> %d\n", u.Username, a.Anime, u.Progress, newProgress)
logOut.WriteLine(fmt.Sprintf("%s progress von %d: %d -> %d", u.Username, a.Anime, u.Progress, newProgress))
err = UpdateUserAnimeProgress(a.Anime, u.MalID, newProgress, updated)
if err != nil {
color.Errorln(err.Error())
logOut.WriteLine(err.Error())
}
}
}
@@ -71,18 +78,21 @@ func LangeArbeit() {
// check if no users registered
if err != nutsdb.ErrBucketEmpty {
color.Errorln(err.Error())
logOut.WriteLine(err.Error())
}
} else {
for _, u := range regUsers {
_, _, err = GetUserData(u.Username)
if err != nil {
color.Errorln(err.Error())
logOut.WriteLine(err.Error())
continue
}
count++
}
}
color.Infof("%d User aktualisiert\n", count)
logOut.WriteLine(fmt.Sprintf("%d User aktualisiert", count))
count = 0
// refresh anime cache with watched
@@ -90,6 +100,7 @@ func LangeArbeit() {
if err != nil {
if err != nutsdb.ErrBucketEmpty {
color.Errorln(err.Error())
logOut.WriteLine(err.Error())
}
} else {
for _, a := range animesUsers {
@@ -101,6 +112,7 @@ func LangeArbeit() {
err = refreshAnime(a.Anime)
if err != nil {
color.Errorln(err.Error())
logOut.WriteLine(err.Error())
continue
}
count++
@@ -108,6 +120,7 @@ func LangeArbeit() {
}
color.Infof("%d Anime aktualisiert\n", count)
logOut.WriteLine(fmt.Sprintf("%d Anime aktualisiert", count))
}
func SehrLangeArbeiten() {
@@ -115,6 +128,7 @@ func SehrLangeArbeiten() {
err := DbClean()
if err != nil {
color.Errorln(err.Error())
logOut.WriteLine(err.Error())
}
}
}

101
ring.go Normal file
View File

@@ -0,0 +1,101 @@
package main
import (
"fmt"
"sync"
"time"
)
// RingBuf implements an indefinitely writable circular buffer with a fixed size. Old data is overridden if write circle loops. Somewhat threadsafe.
type RingBuf struct {
sync.RWMutex
data []byte
writePos int
written int
loop bool
}
// NewRingBuf initializes a new RingBuf with a fixed size > 0
func NewRingBuf(size int) *RingBuf {
if size <= 0 {
return nil
}
rb := &RingBuf{
data: make([]byte, size),
writePos: 0,
written: 0,
loop: false,
}
return rb
}
func (rb *RingBuf) WriteLine(in string) (int, error) {
return rb.Write([]byte(fmt.Sprintf("[%s]: %s\n", time.Now().Format("2006-01-02 15:04:05"), in)))
}
// Write writes all data from input buf to RingBuf, overriding looped data
func (rb *RingBuf) Write(buf []byte) (int, error) {
rb.Lock()
defer rb.Unlock()
inLen := len(buf)
bufLen := len(rb.data)
if !rb.loop && rb.written < bufLen {
rb.written = rb.written + inLen
rb.loop = rb.written >= bufLen
}
// throw away bytes which would get looped over if input bigger than data
if inLen > bufLen {
buf = buf[inLen-bufLen:]
}
// Copy to data field
bytesToEnd := bufLen - rb.writePos
copy(rb.data[rb.writePos:], buf)
if len(buf) > bytesToEnd {
copy(rb.data, buf[bytesToEnd:])
}
// Move writePos
rb.writePos = ((rb.writePos + len(buf)) % bufLen)
return inLen, nil
}
// Size returns the maximum size of the RingBuf
func (rb *RingBuf) Size() int {
return len(rb.data)
}
// Bytes returns content of RingBuf. DON'T WRITE TO SLICE!
func (rb *RingBuf) Bytes() []byte {
rb.RLock()
defer rb.RUnlock()
bufLen := len(rb.data)
if rb.loop {
out := make([]byte, bufLen)
if rb.writePos == 0 {
copy(out, rb.data)
} else {
copy(out, rb.data[rb.writePos:])
copy(out[bufLen-rb.writePos:], rb.data[:rb.writePos])
}
return out
}
out := make([]byte, rb.writePos+1)
copy(out, rb.data[:rb.writePos])
return out
}
// Restart restarts RingBuf from 0
func (rb *RingBuf) Restart() {
rb.writePos = 0
rb.written = 0
rb.loop = false
}
// String returns content of RingBuf as string
func (rb *RingBuf) String() string {
return string(rb.Bytes())
}

6
season.qtpl
View File

@@ -1,5 +1,5 @@
{% package main %}
{% func Index(animes []Anime) %}
{% func Index(animes []Anime, log string) %}
{% collapsespace %}
<!DOCTYPE html>
<html lang="en">
@@ -36,6 +36,10 @@ body { background-color: #1a1a1a; color: #fff; }
</tr>
{% endfor %}
</table>
<h2>Log</h2>
<pre>
{%s log %}
</pre>
</body>
</html>
{% endcollapsespace %}

42
season.qtpl.go
View File

@@ -18,7 +18,7 @@ var (
)
//line season.qtpl:2
func StreamIndex(qw422016 *qt422016.Writer, animes []Anime) {
func StreamIndex(qw422016 *qt422016.Writer, animes []Anime, log string) {
//line season.qtpl:2
qw422016.N().S(`
`)
@@ -47,35 +47,39 @@ func StreamIndex(qw422016 *qt422016.Writer, animes []Anime) {
//line season.qtpl:37
}
//line season.qtpl:37
qw422016.N().S(` </table> </body> </html> `)
qw422016.N().S(` </table> <h2>Log</h2> <pre> `)
//line season.qtpl:41
qw422016.E().S(log)
//line season.qtpl:41
qw422016.N().S(` </pre> </body> </html> `)
//line season.qtpl:45
qw422016.N().S(`
`)
//line season.qtpl:42
//line season.qtpl:46
}
//line season.qtpl:42
func WriteIndex(qq422016 qtio422016.Writer, animes []Anime) {
//line season.qtpl:42
//line season.qtpl:46
func WriteIndex(qq422016 qtio422016.Writer, animes []Anime, log string) {
//line season.qtpl:46
qw422016 := qt422016.AcquireWriter(qq422016)
//line season.qtpl:42
StreamIndex(qw422016, animes)
//line season.qtpl:42
//line season.qtpl:46
StreamIndex(qw422016, animes, log)
//line season.qtpl:46
qt422016.ReleaseWriter(qw422016)
//line season.qtpl:42
//line season.qtpl:46
}
//line season.qtpl:42
func Index(animes []Anime) string {
//line season.qtpl:42
//line season.qtpl:46
func Index(animes []Anime, log string) string {
//line season.qtpl:46
qb422016 := qt422016.AcquireByteBuffer()
//line season.qtpl:42
WriteIndex(qb422016, animes)
//line season.qtpl:42
//line season.qtpl:46
WriteIndex(qb422016, animes, log)
//line season.qtpl:46
qs422016 := string(qb422016.B)
//line season.qtpl:42
//line season.qtpl:46
qt422016.ReleaseByteBuffer(qb422016)
//line season.qtpl:42
//line season.qtpl:46
return qs422016
//line season.qtpl:42
//line season.qtpl:46
}