Revert "Upgrade to go 1.19 and update dependencies (#3069)" (#3085)

This reverts commit bba7c23957.
This commit is contained in:
WithoutPants
2022-11-07 12:33:15 +11:00
committed by GitHub
parent bba7c23957
commit 2609095c7a
939 changed files with 43785 additions and 101302 deletions

View File

@@ -2,12 +2,11 @@ language: go
go:
- 1.x
- tip
before_install:
- go install github.com/mattn/goveralls@latest
install:
- bash .travis.sh
- go get -t ./...
- go get golang.org/x/tools/cmd/cover
- go get github.com/mattn/goveralls
matrix:
allow_failures:
- go: tip

View File

@@ -72,7 +72,11 @@ func (w *BitsWriter) Write(i interface{}) error {
}
}
case []byte:
return w.writeByteSlice(a)
for _, b := range a {
if err := w.writeFullByte(b); err != nil {
return err
}
}
case bool:
if a {
return w.writeBit(1)
@@ -98,21 +102,17 @@ func (w *BitsWriter) Write(i interface{}) error {
// Writes first n bytes of bs if len(bs) > n
// Pads with padByte at the end if len(bs) < n
func (w *BitsWriter) WriteBytesN(bs []byte, n int, padByte uint8) error {
if n == 0 {
return nil
}
if len(bs) >= n {
return w.writeByteSlice(bs[:n])
return w.Write(bs[:n])
}
if err := w.writeByteSlice(bs); err != nil {
if err := w.Write(bs); err != nil {
return err
}
// no bytes.Repeat here to avoid allocation
for i := 0; i < n-len(bs); i++ {
if err := w.writeFullByte(padByte); err != nil {
if err := w.Write(padByte); err != nil {
return err
}
}
@@ -120,42 +120,17 @@ func (w *BitsWriter) WriteBytesN(bs []byte, n int, padByte uint8) error {
return nil
}
func (w *BitsWriter) writeByteSlice(in []byte) error {
if len(in) == 0 {
return nil
}
if w.cacheLen != 0 {
for _, b := range in {
if err := w.writeFullByte(b); err != nil {
func (w *BitsWriter) writeFullInt(in uint64, len int) error {
if w.bo == binary.BigEndian {
for i := len - 1; i >= 0; i-- {
err := w.writeFullByte(byte((in >> (i * 8)) & 0xff))
if err != nil {
return err
}
}
} else {
return w.write(in)
}
return nil
}
func (w *BitsWriter) write(b []byte) error {
if _, err := w.w.Write(b); err != nil {
return err
}
if w.writeCb != nil {
for i := range b {
w.writeCb(b[i : i+1])
}
}
return nil
}
func (w *BitsWriter) writeFullInt(in uint64, len int) error {
if w.bo == binary.BigEndian {
return w.writeBitsN(in, len*8)
} else {
for i := 0; i < len; i++ {
err := w.writeFullByte(byte(in >> (i * 8)))
err := w.writeFullByte(byte((in >> (i * 8)) & 0xff))
if err != nil {
return err
}
@@ -166,7 +141,15 @@ func (w *BitsWriter) writeFullInt(in uint64, len int) error {
}
func (w *BitsWriter) flushBsCache() error {
return w.write(w.bsCache)
if _, err := w.w.Write(w.bsCache); err != nil {
return err
}
if w.writeCb != nil {
w.writeCb(w.bsCache)
}
return nil
}
func (w *BitsWriter) writeFullByte(b byte) error {
@@ -180,9 +163,7 @@ func (w *BitsWriter) writeFullByte(b byte) error {
}
func (w *BitsWriter) writeBit(bit byte) error {
if bit != 0 {
w.cache |= 1 << (7 - w.cacheLen)
}
w.cache = w.cache | (bit)<<(7-w.cacheLen)
w.cacheLen++
if w.cacheLen == 8 {
w.bsCache[0] = w.cache
@@ -196,53 +177,6 @@ func (w *BitsWriter) writeBit(bit byte) error {
return nil
}
func (w *BitsWriter) writeBitsN(toWrite uint64, n int) (err error) {
toWrite &= ^uint64(0) >> (64 - n)
for n > 0 {
if w.cacheLen == 0 {
if n >= 8 {
n -= 8
w.bsCache[0] = byte(toWrite >> n)
if err = w.flushBsCache(); err != nil {
return
}
} else {
w.cacheLen = uint8(n)
w.cache = byte(toWrite << (8 - w.cacheLen))
n = 0
}
} else {
free := int(8 - w.cacheLen)
m := n
if m >= free {
m = free
}
if n <= free {
w.cache |= byte(toWrite << (free - m))
} else {
w.cache |= byte(toWrite >> (n - m))
}
n -= m
w.cacheLen += uint8(m)
if w.cacheLen == 8 {
w.bsCache[0] = w.cache
if err = w.flushBsCache(); err != nil {
return err
}
w.cacheLen = 0
w.cache = 0
}
}
}
return
}
// WriteN writes the input into n bits
func (w *BitsWriter) WriteN(i interface{}, n int) error {
var toWrite uint64
@@ -259,7 +193,13 @@ func (w *BitsWriter) WriteN(i interface{}, n int) error {
return errors.New("astikit: invalid type")
}
return w.writeBitsN(toWrite, n)
for i := n - 1; i >= 0; i-- {
err := w.writeBit(byte(toWrite>>i) & 0x1)
if err != nil {
return err
}
}
return nil
}
// BitsWriterBatch allows to chain multiple Write* calls and check for error only once

View File

@@ -1,8 +0,0 @@
package astikit
func BoolToUInt32(b bool) uint32 {
if b {
return 1
}
return 0
}

View File

@@ -4,56 +4,36 @@ import (
"sync"
)
type CloseFunc func()
type CloseFuncWithError func() error
// CloseFunc is a method that closes something
type CloseFunc func() error
// Closer is an object that can close several things
type Closer struct {
closed bool
fs []CloseFuncWithError
// We need to split into 2 mutexes to allow using .Add() in .Do()
mc *sync.Mutex // Locks .Close()
mf *sync.Mutex // Locks fs
onClosed CloserOnClosed
fs []CloseFunc
m *sync.Mutex
}
type CloserOnClosed func(err error)
// NewCloser creates a new closer
func NewCloser() *Closer {
return &Closer{
mc: &sync.Mutex{},
mf: &sync.Mutex{},
m: &sync.Mutex{},
}
}
// Close implements the io.Closer interface
func (c *Closer) Close() error {
// Lock
c.mc.Lock()
defer c.mc.Unlock()
// Get funcs
c.mf.Lock()
fs := c.fs
c.mf.Unlock()
c.m.Lock()
defer c.m.Unlock()
// Loop through closers
err := NewErrors()
for _, f := range fs {
for _, f := range c.fs {
err.Add(f())
}
// Reset closers
c.fs = []CloseFuncWithError{}
// Update attribute
c.closed = true
// Callback
if c.onClosed != nil {
c.onClosed(err)
}
c.fs = []CloseFunc{}
// Return
if err.IsNil() {
@@ -62,65 +42,16 @@ func (c *Closer) Close() error {
return err
}
// Add adds a close func at the beginning of the list
func (c *Closer) Add(f CloseFunc) {
c.AddWithError(func() error {
f()
return nil
})
}
func (c *Closer) AddWithError(f CloseFuncWithError) {
// Lock
c.mf.Lock()
defer c.mf.Unlock()
// Append
c.fs = append([]CloseFuncWithError{f}, c.fs...)
}
func (c *Closer) Append(dst *Closer) {
// Lock
c.mf.Lock()
dst.mf.Lock()
defer c.mf.Unlock()
defer dst.mf.Unlock()
// Append
c.fs = append(c.fs, dst.fs...)
c.m.Lock()
defer c.m.Unlock()
c.fs = append([]CloseFunc{f}, c.fs...)
}
// NewChild creates a new child closer
func (c *Closer) NewChild() (child *Closer) {
child = NewCloser()
c.AddWithError(child.Close)
c.Add(child.Close)
return
}
// Do executes a callback while ensuring :
// - closer hasn't been closed before
// - closer can't be closed in between
func (c *Closer) Do(fn func()) {
// Lock
c.mc.Lock()
defer c.mc.Unlock()
// Closer already closed
if c.closed {
return
}
// Callback
fn()
}
func (c *Closer) OnClosed(fn CloserOnClosed) {
c.mc.Lock()
defer c.mc.Unlock()
c.onClosed = fn
}
func (c *Closer) IsClosed() bool {
c.mc.Lock()
defer c.mc.Unlock()
return c.closed
}

View File

@@ -7,7 +7,6 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"os"
@@ -156,7 +155,7 @@ func (s *HTTPSender) SendWithTimeout(req *http.Request, timeout time.Duration) (
s.l.Debugf("astikit: sending %s", nr)
if resp, err = s.client.Do(req); err != nil {
// Retry if error is temporary, stop here otherwise
if netError, ok := err.(net.Error); !ok || !netError.Timeout() {
if netError, ok := err.(net.Error); !ok || !netError.Temporary() {
err = fmt.Errorf("astikit: sending %s failed: %w", nr, err)
return
}
@@ -185,20 +184,14 @@ func (s *HTTPSender) SendWithTimeout(req *http.Request, timeout time.Duration) (
return
}
type HTTPSenderHeaderFunc func(h http.Header)
type HTTPSenderStatusCodeFunc func(code int) error
// HTTPSendJSONOptions represents SendJSON options
type HTTPSendJSONOptions struct {
BodyError interface{}
BodyIn interface{}
BodyOut interface{}
HeadersIn map[string]string
HeadersOut HTTPSenderHeaderFunc
Method string
StatusCodeFunc HTTPSenderStatusCodeFunc
URL string
BodyError interface{}
BodyIn interface{}
BodyOut interface{}
Headers map[string]string
Method string
URL string
}
// SendJSON sends a new JSON HTTP request
@@ -222,7 +215,7 @@ func (s *HTTPSender) SendJSON(o HTTPSendJSONOptions) (err error) {
}
// Add headers
for k, v := range o.HeadersIn {
for k, v := range o.Headers {
req.Header.Set(k, v)
}
@@ -234,17 +227,8 @@ func (s *HTTPSender) SendJSON(o HTTPSendJSONOptions) (err error) {
}
defer resp.Body.Close()
// Process headers
if o.HeadersOut != nil {
o.HeadersOut(resp.Header)
}
// Process status code
fn := HTTPSenderDefaultStatusCodeFunc
if o.StatusCodeFunc != nil {
fn = o.StatusCodeFunc
}
if err = fn(resp.StatusCode); err != nil {
if code := resp.StatusCode; code < 200 || code > 299 {
// Try unmarshaling error
if o.BodyError != nil {
if err2 := json.NewDecoder(resp.Body).Decode(o.BodyError); err2 == nil {
@@ -254,35 +238,20 @@ func (s *HTTPSender) SendJSON(o HTTPSendJSONOptions) (err error) {
}
// Default error
err = fmt.Errorf("astikit: validating status code %d failed: %w", resp.StatusCode, err)
err = fmt.Errorf("astikit: invalid status code %d", code)
return
}
// Unmarshal body out
if o.BodyOut != nil {
// Read all
var b []byte
if b, err = ioutil.ReadAll(resp.Body); err != nil {
err = fmt.Errorf("astikit: reading all failed: %w", err)
return
}
// Unmarshal
if err = json.Unmarshal(b, o.BodyOut); err != nil {
err = fmt.Errorf("astikit: unmarshaling failed: %w (json: %s)", err, b)
if err = json.NewDecoder(resp.Body).Decode(o.BodyOut); err != nil {
err = fmt.Errorf("astikit: unmarshaling failed: %w", err)
return
}
}
return
}
func HTTPSenderDefaultStatusCodeFunc(code int) error {
if code < 200 || code > 299 {
return errors.New("astikit: status code should be between 200 and 299")
}
return nil
}
// HTTPResponseFunc is a func that can process an $http.Response
type HTTPResponseFunc func(resp *http.Response) error
@@ -411,7 +380,6 @@ func (d *HTTPDownloader) download(ctx context.Context, srcs []HTTPDownloaderSrc,
}
// Do
//nolint:errcheck
d.l.Do(func() {
// Task is done
defer wg.Done()

View File

@@ -1,35 +0,0 @@
package astikit
import (
"bytes"
"encoding/json"
"fmt"
)
func JSONEqual(a, b interface{}) bool {
ba, err := json.Marshal(a)
if err != nil {
return false
}
bb, err := json.Marshal(b)
if err != nil {
return false
}
return bytes.Equal(ba, bb)
}
func JSONClone(src, dst interface{}) (err error) {
// Marshal
var b []byte
if b, err = json.Marshal(src); err != nil {
err = fmt.Errorf("main: marshaling failed: %w", err)
return
}
// Unmarshal
if err = json.Unmarshal(b, dst); err != nil {
err = fmt.Errorf("main: unmarshaling failed: %w", err)
return
}
return
}

View File

@@ -4,66 +4,11 @@ import (
"context"
)
// LoggerLevel represents a logger level
type LoggerLevel int
// Logger levels
const (
LoggerLevelDebug LoggerLevel = iota
LoggerLevelInfo
LoggerLevelWarn
LoggerLevelError
LoggerLevelFatal
)
// LoggerLevelFromString creates a logger level from string
func LoggerLevelFromString(s string) LoggerLevel {
switch s {
case "debug":
return LoggerLevelDebug
case "error":
return LoggerLevelError
case "fatal":
return LoggerLevelFatal
case "warn":
return LoggerLevelWarn
default:
return LoggerLevelInfo
}
}
func (l LoggerLevel) String() string {
switch l {
case LoggerLevelDebug:
return "debug"
case LoggerLevelError:
return "error"
case LoggerLevelFatal:
return "fatal"
case LoggerLevelWarn:
return "warn"
default:
return "info"
}
}
func (l *LoggerLevel) UnmarshalText(b []byte) error {
*l = LoggerLevelFromString(string(b))
return nil
}
func (l LoggerLevel) MarshalText() ([]byte, error) {
b := []byte(l.String())
return b, nil
}
// CompleteLogger represents a complete logger
type CompleteLogger interface {
SeverityCtxLogger
SeverityLogger
SeverityWriteLogger
SeverityWriteCtxLogger
StdLogger
SeverityLogger
SeverityCtxLogger
}
// StdLogger represents a standard logger
@@ -86,15 +31,6 @@ type SeverityLogger interface {
Warnf(format string, v ...interface{})
}
type TestLogger interface {
Error(v ...interface{})
Errorf(format string, v ...interface{})
Fatal(v ...interface{})
Fatalf(format string, v ...interface{})
Log(v ...interface{})
Logf(format string, v ...interface{})
}
// SeverityCtxLogger represents a severity with context logger
type SeverityCtxLogger interface {
DebugC(ctx context.Context, v ...interface{})
@@ -109,108 +45,38 @@ type SeverityCtxLogger interface {
WarnCf(ctx context.Context, format string, v ...interface{})
}
type SeverityWriteLogger interface {
Write(l LoggerLevel, v ...interface{})
Writef(l LoggerLevel, format string, v ...interface{})
}
type SeverityWriteCtxLogger interface {
WriteC(ctx context.Context, l LoggerLevel, v ...interface{})
WriteCf(ctx context.Context, l LoggerLevel, format string, v ...interface{})
}
type completeLogger struct {
print, debug, error, fatal, info, warn func(v ...interface{})
printf, debugf, errorf, fatalf, infof, warnf func(format string, v ...interface{})
debugC, errorC, fatalC, infoC, warnC func(ctx context.Context, v ...interface{})
debugCf, errorCf, fatalCf, infoCf, warnCf func(ctx context.Context, format string, v ...interface{})
write func(l LoggerLevel, v ...interface{})
writeC func(ctx context.Context, l LoggerLevel, v ...interface{})
writeCf func(ctx context.Context, l LoggerLevel, format string, v ...interface{})
writef func(l LoggerLevel, format string, v ...interface{})
}
func newCompleteLogger() *completeLogger {
l := &completeLogger{}
l.debug = func(v ...interface{}) { l.print(v...) }
l.debugf = func(format string, v ...interface{}) { l.printf(format, v...) }
l.debugC = func(ctx context.Context, v ...interface{}) { l.debug(v...) }
l.debugCf = func(ctx context.Context, format string, v ...interface{}) { l.debugf(format, v...) }
l.error = func(v ...interface{}) { l.print(v...) }
l.errorf = func(format string, v ...interface{}) { l.printf(format, v...) }
l.errorC = func(ctx context.Context, v ...interface{}) { l.error(v...) }
l.errorCf = func(ctx context.Context, format string, v ...interface{}) { l.errorf(format, v...) }
l.fatal = func(v ...interface{}) { l.print(v...) }
l.fatalf = func(format string, v ...interface{}) { l.printf(format, v...) }
l.fatalC = func(ctx context.Context, v ...interface{}) { l.fatal(v...) }
l.fatalCf = func(ctx context.Context, format string, v ...interface{}) { l.fatalf(format, v...) }
l.info = func(v ...interface{}) { l.print(v...) }
l.infof = func(format string, v ...interface{}) { l.printf(format, v...) }
l.infoC = func(ctx context.Context, v ...interface{}) { l.info(v...) }
l.infoCf = func(ctx context.Context, format string, v ...interface{}) { l.infof(format, v...) }
l.print = func(v ...interface{}) {}
l.printf = func(format string, v ...interface{}) {}
l.warn = func(v ...interface{}) { l.print(v...) }
l.warnf = func(format string, v ...interface{}) { l.printf(format, v...) }
l.warnC = func(ctx context.Context, v ...interface{}) { l.warn(v...) }
l.warnCf = func(ctx context.Context, format string, v ...interface{}) { l.warnf(format, v...) }
l.write = func(lv LoggerLevel, v ...interface{}) {
switch lv {
case LoggerLevelDebug:
l.debug(v...)
case LoggerLevelError:
l.error(v...)
case LoggerLevelFatal:
l.fatal(v...)
case LoggerLevelWarn:
l.warn(v...)
default:
l.info(v...)
}
return &completeLogger{
debug: func(v ...interface{}) {},
debugf: func(format string, v ...interface{}) {},
debugC: func(ctx context.Context, v ...interface{}) {},
debugCf: func(ctx context.Context, format string, v ...interface{}) {},
error: func(v ...interface{}) {},
errorf: func(format string, v ...interface{}) {},
errorC: func(ctx context.Context, v ...interface{}) {},
errorCf: func(ctx context.Context, format string, v ...interface{}) {},
fatal: func(v ...interface{}) {},
fatalf: func(format string, v ...interface{}) {},
fatalC: func(ctx context.Context, v ...interface{}) {},
fatalCf: func(ctx context.Context, format string, v ...interface{}) {},
info: func(v ...interface{}) {},
infof: func(format string, v ...interface{}) {},
infoC: func(ctx context.Context, v ...interface{}) {},
infoCf: func(ctx context.Context, format string, v ...interface{}) {},
print: func(v ...interface{}) {},
printf: func(format string, v ...interface{}) {},
warn: func(v ...interface{}) {},
warnf: func(format string, v ...interface{}) {},
warnC: func(ctx context.Context, v ...interface{}) {},
warnCf: func(ctx context.Context, format string, v ...interface{}) {},
}
l.writeC = func(ctx context.Context, lv LoggerLevel, v ...interface{}) {
switch lv {
case LoggerLevelDebug:
l.debugC(ctx, v...)
case LoggerLevelError:
l.errorC(ctx, v...)
case LoggerLevelFatal:
l.fatalC(ctx, v...)
case LoggerLevelWarn:
l.warnC(ctx, v...)
default:
l.infoC(ctx, v...)
}
}
l.writeCf = func(ctx context.Context, lv LoggerLevel, format string, v ...interface{}) {
switch lv {
case LoggerLevelDebug:
l.debugCf(ctx, format, v...)
case LoggerLevelError:
l.errorCf(ctx, format, v...)
case LoggerLevelFatal:
l.fatalCf(ctx, format, v...)
case LoggerLevelWarn:
l.warnCf(ctx, format, v...)
default:
l.infoCf(ctx, format, v...)
}
}
l.writef = func(lv LoggerLevel, format string, v ...interface{}) {
switch lv {
case LoggerLevelDebug:
l.debugf(format, v...)
case LoggerLevelError:
l.errorf(format, v...)
case LoggerLevelFatal:
l.fatalf(format, v...)
case LoggerLevelWarn:
l.warnf(format, v...)
default:
l.infof(format, v...)
}
}
return l
}
func (l *completeLogger) Debug(v ...interface{}) { l.debug(v...) }
@@ -245,16 +111,6 @@ func (l *completeLogger) WarnC(ctx context.Context, v ...interface{}) { l.warnC(
func (l *completeLogger) WarnCf(ctx context.Context, format string, v ...interface{}) {
l.warnCf(ctx, format, v...)
}
func (l *completeLogger) Write(lv LoggerLevel, v ...interface{}) { l.write(lv, v...) }
func (l *completeLogger) Writef(lv LoggerLevel, format string, v ...interface{}) {
l.writef(lv, format, v...)
}
func (l *completeLogger) WriteC(ctx context.Context, lv LoggerLevel, v ...interface{}) {
l.writeC(ctx, lv, v...)
}
func (l *completeLogger) WriteCf(ctx context.Context, lv LoggerLevel, format string, v ...interface{}) {
l.writeCf(ctx, lv, format, v...)
}
// AdaptStdLogger transforms an StdLogger into a CompleteLogger if needed
func AdaptStdLogger(i StdLogger) CompleteLogger {
@@ -278,6 +134,15 @@ func AdaptStdLogger(i StdLogger) CompleteLogger {
l.infof = v.Infof
l.warn = v.Warn
l.warnf = v.Warnf
} else {
l.debug = l.print
l.debugf = l.printf
l.error = l.print
l.errorf = l.printf
l.info = l.print
l.infof = l.printf
l.warn = l.print
l.warnf = l.printf
}
if v, ok := i.(SeverityCtxLogger); ok {
l.debugC = v.DebugC
@@ -290,38 +155,17 @@ func AdaptStdLogger(i StdLogger) CompleteLogger {
l.infoCf = v.InfoCf
l.warnC = v.WarnC
l.warnCf = v.WarnCf
}
if v, ok := i.(SeverityWriteLogger); ok {
l.write = v.Write
l.writef = v.Writef
}
if v, ok := i.(SeverityWriteCtxLogger); ok {
l.writeC = v.WriteC
l.writeCf = v.WriteCf
} else {
l.debugC = func(ctx context.Context, v ...interface{}) { l.debug(v...) }
l.debugCf = func(ctx context.Context, format string, v ...interface{}) { l.debugf(format, v...) }
l.errorC = func(ctx context.Context, v ...interface{}) { l.error(v...) }
l.errorCf = func(ctx context.Context, format string, v ...interface{}) { l.errorf(format, v...) }
l.fatalC = func(ctx context.Context, v ...interface{}) { l.fatal(v...) }
l.fatalCf = func(ctx context.Context, format string, v ...interface{}) { l.fatalf(format, v...) }
l.infoC = func(ctx context.Context, v ...interface{}) { l.info(v...) }
l.infoCf = func(ctx context.Context, format string, v ...interface{}) { l.infof(format, v...) }
l.warnC = func(ctx context.Context, v ...interface{}) { l.warn(v...) }
l.warnCf = func(ctx context.Context, format string, v ...interface{}) { l.warnf(format, v...) }
}
return l
}
// AdaptTestLogger transforms a TestLogger into a CompleteLogger if needed
func AdaptTestLogger(i TestLogger) CompleteLogger {
if v, ok := i.(CompleteLogger); ok {
return v
}
l := newCompleteLogger()
if i == nil {
return l
}
l.error = i.Error
l.errorf = i.Errorf
l.fatal = i.Fatal
l.fatalf = i.Fatalf
l.print = i.Log
l.printf = i.Logf
l.debug = l.print
l.debugf = l.printf
l.info = l.print
l.infof = l.printf
l.warn = l.print
l.warnf = l.printf
return l
}

View File

@@ -6,7 +6,6 @@ import (
"io"
"os"
"path/filepath"
"strings"
)
// SSHSession represents an SSH Session
@@ -28,9 +27,6 @@ func SSHCopyFileFunc(fn SSHSessionFunc) CopyFileFunc {
return
}
// Escape dir path
d := strings.ReplaceAll(filepath.Dir(dst), " ", "\\ ")
// Using local closure allows better readibility for the defer c.Close() since it
// isolates the use of the ssh session
if err = func() (err error) {
@@ -44,7 +40,7 @@ func SSHCopyFileFunc(fn SSHSessionFunc) CopyFileFunc {
defer c.Close()
// Create the destination folder
if err = s.Run("mkdir -p " + d); err != nil {
if err = s.Run("mkdir -p " + filepath.Dir(dst)); err != nil {
err = fmt.Errorf("astikit: creating %s failed: %w", filepath.Dir(dst), err)
return
}
@@ -74,7 +70,7 @@ func SSHCopyFileFunc(fn SSHSessionFunc) CopyFileFunc {
defer stdin.Close()
// Use "scp" command
if err = s.Start("scp -qt " + d); err != nil {
if err = s.Start("scp -qt \"" + filepath.Dir(dst) + "\""); err != nil {
err = fmt.Errorf("astikit: scp to %s failed: %w", dst, err)
return
}

View File

@@ -20,9 +20,8 @@ type Stater struct {
// StatOptions represents stat options
type StatOptions struct {
Handler StatHandler
Metadata *StatMetadata
// Either a StatValuer or StatValuerOverTime
Valuer interface{}
}
// StatsHandleFunc is a method that can handle stat values
@@ -36,13 +35,10 @@ type StatMetadata struct {
Unit string
}
// StatValuer represents a stat valuer
type StatValuer interface {
Value() interface{}
}
// StatValuerOverTime represents a stat valuer over time
type StatValuerOverTime interface {
// StatHandler represents a stat handler
type StatHandler interface {
Start()
Stop()
Value(delta time.Duration) interface{}
}
@@ -100,21 +96,10 @@ func (s *Stater) Start(ctx context.Context) {
// Loop through stats
var stats []StatValue
s.m.Lock()
for _, o := range s.ss {
// Get value
var v interface{}
if h, ok := o.Valuer.(StatValuer); ok {
v = h.Value()
} else if h, ok := o.Valuer.(StatValuerOverTime); ok {
v = h.Value(delta)
} else {
continue
}
// Append
for _, v := range s.ss {
stats = append(stats, StatValue{
StatMetadata: o.Metadata,
Value: v,
StatMetadata: v.Metadata,
Value: v.Handler.Value(delta),
})
}
s.m.Unlock()
@@ -153,34 +138,41 @@ func (s *Stater) DelStats(os ...StatOptions) {
}
}
type durationStatOverTime struct {
d time.Duration
fn func(d, delta time.Duration) interface{}
m *sync.Mutex // Locks isStarted
lastBeginAt time.Time
type durationStat struct {
d time.Duration
fn func(d, delta time.Duration) interface{}
isStarted bool
m *sync.Mutex // Locks isStarted
startedAt time.Time
}
func newDurationStatOverTime(fn func(d, delta time.Duration) interface{}) *durationStatOverTime {
return &durationStatOverTime{
func newDurationStat(fn func(d, delta time.Duration) interface{}) *durationStat {
return &durationStat{
fn: fn,
m: &sync.Mutex{},
}
}
func (s *durationStatOverTime) Begin() {
func (s *durationStat) Begin() {
s.m.Lock()
defer s.m.Unlock()
s.lastBeginAt = now()
if !s.isStarted {
return
}
s.startedAt = now()
}
func (s *durationStatOverTime) End() {
func (s *durationStat) End() {
s.m.Lock()
defer s.m.Unlock()
s.d += now().Sub(s.lastBeginAt)
s.lastBeginAt = time.Time{}
if !s.isStarted {
return
}
s.d += now().Sub(s.startedAt)
s.startedAt = time.Time{}
}
func (s *durationStatOverTime) Value(delta time.Duration) (o interface{}) {
func (s *durationStat) Value(delta time.Duration) (o interface{}) {
// Lock
s.m.Lock()
defer s.m.Unlock()
@@ -190,9 +182,9 @@ func (s *durationStatOverTime) Value(delta time.Duration) (o interface{}) {
d := s.d
// Recording is still in process
if !s.lastBeginAt.IsZero() {
d += n.Sub(s.lastBeginAt)
s.lastBeginAt = n
if !s.startedAt.IsZero() {
d += n.Sub(s.startedAt)
s.startedAt = n
}
// Compute stat
@@ -201,14 +193,27 @@ func (s *durationStatOverTime) Value(delta time.Duration) (o interface{}) {
return
}
func (s *durationStat) Start() {
s.m.Lock()
defer s.m.Unlock()
s.d = 0
s.isStarted = true
}
func (s *durationStat) Stop() {
s.m.Lock()
defer s.m.Unlock()
s.isStarted = false
}
// DurationPercentageStat is an object capable of computing the percentage of time some work is taking per second
type DurationPercentageStat struct {
*durationStatOverTime
*durationStat
}
// NewDurationPercentageStat creates a new duration percentage stat
func NewDurationPercentageStat() *DurationPercentageStat {
return &DurationPercentageStat{durationStatOverTime: newDurationStatOverTime(func(d, delta time.Duration) interface{} {
return &DurationPercentageStat{durationStat: newDurationStat(func(d, delta time.Duration) interface{} {
if delta == 0 {
return 0
}
@@ -216,28 +221,46 @@ func NewDurationPercentageStat() *DurationPercentageStat {
})}
}
type counterStatOverTime struct {
c float64
fn func(c, t float64, delta time.Duration) interface{}
m *sync.Mutex // Locks isStarted
t float64
type counterStat struct {
c float64
fn func(c, t float64, delta time.Duration) interface{}
isStarted bool
m *sync.Mutex // Locks isStarted
t float64
}
func newCounterStatOverTime(fn func(c, t float64, delta time.Duration) interface{}) *counterStatOverTime {
return &counterStatOverTime{
func newCounterStat(fn func(c, t float64, delta time.Duration) interface{}) *counterStat {
return &counterStat{
fn: fn,
m: &sync.Mutex{},
}
}
func (s *counterStatOverTime) Add(delta float64) {
func (s *counterStat) Add(delta float64) {
s.m.Lock()
defer s.m.Unlock()
if !s.isStarted {
return
}
s.c += delta
s.t++
}
func (s *counterStatOverTime) Value(delta time.Duration) interface{} {
func (s *counterStat) Start() {
s.m.Lock()
defer s.m.Unlock()
s.c = 0
s.isStarted = true
s.t = 0
}
func (s *counterStat) Stop() {
s.m.Lock()
defer s.m.Unlock()
s.isStarted = true
}
func (s *counterStat) Value(delta time.Duration) interface{} {
s.m.Lock()
defer s.m.Unlock()
c := s.c
@@ -249,12 +272,12 @@ func (s *counterStatOverTime) Value(delta time.Duration) interface{} {
// CounterAvgStat is an object capable of computing the average value of a counter
type CounterAvgStat struct {
*counterStatOverTime
*counterStat
}
// NewCounterAvgStat creates a new counter avg stat
func NewCounterAvgStat() *CounterAvgStat {
return &CounterAvgStat{counterStatOverTime: newCounterStatOverTime(func(c, t float64, delta time.Duration) interface{} {
return &CounterAvgStat{counterStat: newCounterStat(func(c, t float64, delta time.Duration) interface{} {
if t == 0 {
return 0
}
@@ -264,38 +287,15 @@ func NewCounterAvgStat() *CounterAvgStat {
// CounterRateStat is an object capable of computing the average value of a counter per second
type CounterRateStat struct {
*counterStatOverTime
*counterStat
}
// NewCounterRateStat creates a new counter rate stat
func NewCounterRateStat() *CounterRateStat {
return &CounterRateStat{counterStatOverTime: newCounterStatOverTime(func(c, t float64, delta time.Duration) interface{} {
return &CounterRateStat{counterStat: newCounterStat(func(c, t float64, delta time.Duration) interface{} {
if delta.Seconds() == 0 {
return 0
}
return c / delta.Seconds()
})}
}
// CounterStat is an object capable of computing a counter that never gets reset
type CounterStat struct {
c float64
m *sync.Mutex
}
// NewCounterStat creates a new counter stat
func NewCounterStat() *CounterStat {
return &CounterStat{m: &sync.Mutex{}}
}
func (s *CounterStat) Add(delta float64) {
s.m.Lock()
defer s.m.Unlock()
s.c += delta
}
func (s *CounterStat) Value() interface{} {
s.m.Lock()
defer s.m.Unlock()
return s.c
}

View File

@@ -209,13 +209,13 @@ func (c *Chan) Stats() []StatOptions {
}
return []StatOptions{
{
Handler: c.statWorkRatio,
Metadata: &StatMetadata{
Description: "Percentage of time doing work",
Label: "Work ratio",
Name: StatNameWorkRatio,
Unit: "%",
},
Valuer: c.statWorkRatio,
},
}
}

View File

@@ -7,42 +7,32 @@ import (
"net/http"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
)
// Translator represents an object capable of translating stuff
type Translator struct {
defaultLanguage string
m *sync.RWMutex // Lock p
p map[string]string
validLanguages map[string]bool
m *sync.RWMutex // Lock p
o TranslatorOptions
p map[string]string
}
// TranslatorOptions represents Translator options
type TranslatorOptions struct {
DefaultLanguage string
ValidLanguages []string
}
// NewTranslator creates a new Translator
func NewTranslator(o TranslatorOptions) (t *Translator) {
t = &Translator{
defaultLanguage: o.DefaultLanguage,
m: &sync.RWMutex{},
p: make(map[string]string),
validLanguages: make(map[string]bool),
func NewTranslator(o TranslatorOptions) *Translator {
return &Translator{
m: &sync.RWMutex{},
o: o,
p: make(map[string]string),
}
for _, l := range o.ValidLanguages {
t.validLanguages[l] = true
}
return
}
// ParseDir adds translations located in ".json" files in the specified dir
// If ".json" files are located in child dirs, keys will be prefixed with their paths
func (t *Translator) ParseDir(dirPath string) (err error) {
// Default dir path
if dirPath == "" {
@@ -60,8 +50,11 @@ func (t *Translator) ParseDir(dirPath string) (err error) {
return
}
// Only process files
// Only process first level files
if info.IsDir() {
if path != dirPath {
err = filepath.SkipDir
}
return
}
@@ -71,7 +64,7 @@ func (t *Translator) ParseDir(dirPath string) (err error) {
}
// Parse file
if err = t.ParseFile(dirPath, path); err != nil {
if err = t.ParseFile(path); err != nil {
err = fmt.Errorf("astikit: parsing %s failed: %w", path, err)
return
}
@@ -84,7 +77,7 @@ func (t *Translator) ParseDir(dirPath string) (err error) {
}
// ParseFile adds translation located in the provided path
func (t *Translator) ParseFile(dirPath, path string) (err error) {
func (t *Translator) ParseFile(path string) (err error) {
// Lock
t.m.Lock()
defer t.m.Unlock()
@@ -104,26 +97,8 @@ func (t *Translator) ParseFile(dirPath, path string) (err error) {
return
}
// Get language
language := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
// Update valid languages
t.validLanguages[language] = true
// Get prefix
prefix := language
if dp := filepath.Dir(path); dp != dirPath {
var fs []string
for _, v := range strings.Split(strings.TrimPrefix(dp, dirPath), string(os.PathSeparator)) {
if v != "" {
fs = append(fs, v)
}
}
prefix += "." + strings.Join(fs, ".")
}
// Parse
t.parse(p, prefix)
t.parse(p, strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)))
return
}
@@ -148,7 +123,7 @@ func (t *Translator) HTTPMiddleware(h http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
// Store language in context
if l := r.Header.Get("Accept-Language"); l != "" {
*r = *r.WithContext(contextWithTranslatorLanguage(r.Context(), t.parseAcceptLanguage(l)))
*r = *r.WithContext(contextWithTranslatorLanguage(r.Context(), l))
}
// Next handler
@@ -156,56 +131,7 @@ func (t *Translator) HTTPMiddleware(h http.Handler) http.Handler {
})
}
func (t *Translator) parseAcceptLanguage(h string) string {
// Split on comma
var qs []float64
ls := make(map[float64][]string)
for _, c := range strings.Split(strings.TrimSpace(h), ",") {
// Empty
c = strings.TrimSpace(c)
if c == "" {
continue
}
// Split on semi colon
ss := strings.Split(c, ";")
// Parse coefficient
q := float64(1)
if len(ss) > 1 {
s := strings.TrimSpace(ss[1])
if strings.HasPrefix(s, "q=") {
var err error
if q, err = strconv.ParseFloat(strings.TrimPrefix(s, "q="), 64); err != nil {
q = 1
}
}
}
// Add
if _, ok := ls[q]; !ok {
qs = append(qs, q)
}
ls[q] = append(ls[q], strings.TrimSpace(ss[0]))
}
// Order coefficients
sort.Float64s(qs)
// Loop through coefficients in reverse order
for idx := len(qs) - 1; idx >= 0; idx-- {
for _, l := range ls[qs[idx]] {
if _, ok := t.validLanguages[l]; ok {
return l
}
}
}
return ""
}
const contextKeyTranslatorLanguage = contextKey("astikit.translator.language")
type contextKey string
const contextKeyTranslatorLanguage = "astikit.translator.language"
func contextWithTranslatorLanguage(ctx context.Context, language string) context.Context {
return context.WithValue(ctx, contextKeyTranslatorLanguage, language)
@@ -221,7 +147,7 @@ func translatorLanguageFromContext(ctx context.Context) string {
func (t *Translator) language(language string) string {
if language == "" {
return t.defaultLanguage
return t.o.DefaultLanguage
}
return language
}
@@ -245,28 +171,14 @@ func (t *Translator) Translate(language, key string) string {
}
// Default translation
k2 := t.key(t.defaultLanguage, key)
k2 := t.key(t.o.DefaultLanguage, key)
if v, ok = t.p[k2]; ok {
return v
}
return k1
}
// Translatef translates a key into a specific language with optional formatting args
func (t *Translator) Translatef(language, key string, args ...interface{}) string {
return fmt.Sprintf(t.Translate(language, key), args...)
}
// TranslateCtx is an alias for TranslateC
// TranslateCtx translates a key using the language specified in the context
func (t *Translator) TranslateCtx(ctx context.Context, key string) string {
return t.TranslateC(ctx, key)
}
// TranslateC translates a key using the language specified in the context
func (t *Translator) TranslateC(ctx context.Context, key string) string {
return t.Translate(translatorLanguageFromContext(ctx), key)
}
func (t *Translator) TranslateCf(ctx context.Context, key string, args ...interface{}) string {
return t.Translatef(translatorLanguageFromContext(ctx), key, args...)
}

View File

@@ -6,7 +6,6 @@ install:
- go get -t ./...
- go get golang.org/x/tools/cmd/cover
- go get github.com/mattn/goveralls
- go install github.com/mattn/goveralls
matrix:
allow_failures:
- go: tip

View File

@@ -7,7 +7,7 @@ This is a Golang library to manipulate subtitles.
It allows you to manipulate `srt`, `stl`, `ttml`, `ssa/ass`, `webvtt` and `teletext` files for now.
Available operations are `parsing`, `writing`, `applying linear correction`, `syncing`, `fragmenting`, `unfragmenting`, `merging` and `optimizing`.
Available operations are `parsing`, `writing`, `syncing`, `fragmenting`, `unfragmenting`, `merging` and `optimizing`.
# Installation
@@ -43,9 +43,6 @@ s1.Optimize()
// Unfragment the subtitles
s1.Unfragment()
// Apply linear correction
s1.ApplyLinearCorrection(1*time.Second, 2*time.Second, 5*time.Second, 7*time.Second)
// Write subtitles
s1.Write("/path/to/example.srt")
var buf = &bytes.Buffer{}
@@ -60,10 +57,6 @@ If **astisub** has been installed properly you can:
astisub convert -i example.srt -o example.ttml
- apply linear correction to any type of subtitle:
astisub apply-linear-correction -i example.srt -a1 1s -d1 2s -a2 5s -d2 7s -o example.out.srt
- fragment any type of subtitle:
astisub fragment -i example.srt -f 2s -o example.out.srt
@@ -93,7 +86,6 @@ If **astisub** has been installed properly you can:
- [x] merging
- [x] ordering
- [x] optimizing
- [x] linear correction
- [x] .srt
- [x] .ttml
- [x] .vtt

View File

@@ -640,19 +640,6 @@ func (s *Subtitles) Unfragment() {
}
}
// ApplyLinearCorrection applies linear correction
func (s *Subtitles) ApplyLinearCorrection(actual1, desired1, actual2, desired2 time.Duration) {
// Get parameters
a := float64(desired2-desired1) / float64(actual2-actual1)
b := time.Duration(float64(desired1) - a*float64(actual1))
// Loop through items
for idx := range s.Items {
s.Items[idx].EndAt = time.Duration(a*float64(s.Items[idx].EndAt)) + b
s.Items[idx].StartAt = time.Duration(a*float64(s.Items[idx].StartAt)) + b
}
}
// Write writes subtitles to a file
func (s Subtitles) Write(dst string) (err error) {
// Create the file

View File

@@ -6,7 +6,6 @@ install:
- go get -t ./...
- go get golang.org/x/tools/cmd/cover
- go get github.com/mattn/goveralls
- go install github.com/mattn/goveralls
matrix:
allow_failures:
- go: tip

View File

@@ -12,10 +12,6 @@ WARNING: this library is not yet production ready. Use at your own risks!
To install the library use the following:
go get -u github.com/asticode/go-astits/...
To install the executables use the following:
go install github.com/asticode/go-astits/cmd
# Before looking at the code...

View File

@@ -4,15 +4,22 @@ const (
crc32Polynomial = uint32(0xffffffff)
)
// computeCRC32 computes a CRC32
// https://stackoverflow.com/questions/35034042/how-to-calculate-crc32-in-psi-si-packet
func computeCRC32(bs []byte) uint32 {
return updateCRC32(crc32Polynomial, bs)
}
// Based on VLC implementation using a static CRC table (1kb additional memory on start, without
// reallocations): https://github.com/videolan/vlc/blob/master/modules/mux/mpeg/ps.c
func updateCRC32(crc32 uint32, bs []byte) uint32 {
for _, b := range bs {
crc32 = (crc32 << 8) ^ tableCRC32[((crc32>>24)^uint32(b))&0xff]
for i := 0; i < 8; i++ {
if (crc32 >= uint32(0x80000000)) != (b >= uint8(0x80)) {
crc32 = (crc32 << 1) ^ 0x04C11DB7
} else {
crc32 = crc32 << 1
}
b <<= 1
}
}
return crc32
}

View File

@@ -1,37 +0,0 @@
// Code generated by astits using internal/cmd/crc32_table. DO NOT EDIT
package astits
var tableCRC32 = [256]uint32{
0x00000000, 0x04C11DB7, 0x09823B6E, 0x0D4326D9, 0x130476DC, 0x17C56B6B, 0x1A864DB2, 0x1E475005,
0x2608EDB8, 0x22C9F00F, 0x2F8AD6D6, 0x2B4BCB61, 0x350C9B64, 0x31CD86D3, 0x3C8EA00A, 0x384FBDBD,
0x4C11DB70, 0x48D0C6C7, 0x4593E01E, 0x4152FDA9, 0x5F15ADAC, 0x5BD4B01B, 0x569796C2, 0x52568B75,
0x6A1936C8, 0x6ED82B7F, 0x639B0DA6, 0x675A1011, 0x791D4014, 0x7DDC5DA3, 0x709F7B7A, 0x745E66CD,
0x9823B6E0, 0x9CE2AB57, 0x91A18D8E, 0x95609039, 0x8B27C03C, 0x8FE6DD8B, 0x82A5FB52, 0x8664E6E5,
0xBE2B5B58, 0xBAEA46EF, 0xB7A96036, 0xB3687D81, 0xAD2F2D84, 0xA9EE3033, 0xA4AD16EA, 0xA06C0B5D,
0xD4326D90, 0xD0F37027, 0xDDB056FE, 0xD9714B49, 0xC7361B4C, 0xC3F706FB, 0xCEB42022, 0xCA753D95,
0xF23A8028, 0xF6FB9D9F, 0xFBB8BB46, 0xFF79A6F1, 0xE13EF6F4, 0xE5FFEB43, 0xE8BCCD9A, 0xEC7DD02D,
0x34867077, 0x30476DC0, 0x3D044B19, 0x39C556AE, 0x278206AB, 0x23431B1C, 0x2E003DC5, 0x2AC12072,
0x128E9DCF, 0x164F8078, 0x1B0CA6A1, 0x1FCDBB16, 0x018AEB13, 0x054BF6A4, 0x0808D07D, 0x0CC9CDCA,
0x7897AB07, 0x7C56B6B0, 0x71159069, 0x75D48DDE, 0x6B93DDDB, 0x6F52C06C, 0x6211E6B5, 0x66D0FB02,
0x5E9F46BF, 0x5A5E5B08, 0x571D7DD1, 0x53DC6066, 0x4D9B3063, 0x495A2DD4, 0x44190B0D, 0x40D816BA,
0xACA5C697, 0xA864DB20, 0xA527FDF9, 0xA1E6E04E, 0xBFA1B04B, 0xBB60ADFC, 0xB6238B25, 0xB2E29692,
0x8AAD2B2F, 0x8E6C3698, 0x832F1041, 0x87EE0DF6, 0x99A95DF3, 0x9D684044, 0x902B669D, 0x94EA7B2A,
0xE0B41DE7, 0xE4750050, 0xE9362689, 0xEDF73B3E, 0xF3B06B3B, 0xF771768C, 0xFA325055, 0xFEF34DE2,
0xC6BCF05F, 0xC27DEDE8, 0xCF3ECB31, 0xCBFFD686, 0xD5B88683, 0xD1799B34, 0xDC3ABDED, 0xD8FBA05A,
0x690CE0EE, 0x6DCDFD59, 0x608EDB80, 0x644FC637, 0x7A089632, 0x7EC98B85, 0x738AAD5C, 0x774BB0EB,
0x4F040D56, 0x4BC510E1, 0x46863638, 0x42472B8F, 0x5C007B8A, 0x58C1663D, 0x558240E4, 0x51435D53,
0x251D3B9E, 0x21DC2629, 0x2C9F00F0, 0x285E1D47, 0x36194D42, 0x32D850F5, 0x3F9B762C, 0x3B5A6B9B,
0x0315D626, 0x07D4CB91, 0x0A97ED48, 0x0E56F0FF, 0x1011A0FA, 0x14D0BD4D, 0x19939B94, 0x1D528623,
0xF12F560E, 0xF5EE4BB9, 0xF8AD6D60, 0xFC6C70D7, 0xE22B20D2, 0xE6EA3D65, 0xEBA91BBC, 0xEF68060B,
0xD727BBB6, 0xD3E6A601, 0xDEA580D8, 0xDA649D6F, 0xC423CD6A, 0xC0E2D0DD, 0xCDA1F604, 0xC960EBB3,
0xBD3E8D7E, 0xB9FF90C9, 0xB4BCB610, 0xB07DABA7, 0xAE3AFBA2, 0xAAFBE615, 0xA7B8C0CC, 0xA379DD7B,
0x9B3660C6, 0x9FF77D71, 0x92B45BA8, 0x9675461F, 0x8832161A, 0x8CF30BAD, 0x81B02D74, 0x857130C3,
0x5D8A9099, 0x594B8D2E, 0x5408ABF7, 0x50C9B640, 0x4E8EE645, 0x4A4FFBF2, 0x470CDD2B, 0x43CDC09C,
0x7B827D21, 0x7F436096, 0x7200464F, 0x76C15BF8, 0x68860BFD, 0x6C47164A, 0x61043093, 0x65C52D24,
0x119B4BE9, 0x155A565E, 0x18197087, 0x1CD86D30, 0x029F3D35, 0x065E2082, 0x0B1D065B, 0x0FDC1BEC,
0x3793A651, 0x3352BBE6, 0x3E119D3F, 0x3AD08088, 0x2497D08D, 0x2056CD3A, 0x2D15EBE3, 0x29D4F654,
0xC5A92679, 0xC1683BCE, 0xCC2B1D17, 0xC8EA00A0, 0xD6AD50A5, 0xD26C4D12, 0xDF2F6BCB, 0xDBEE767C,
0xE3A1CBC1, 0xE760D676, 0xEA23F0AF, 0xEEE2ED18, 0xF0A5BD1D, 0xF464A0AA, 0xF9278673, 0xFDE69BC4,
0x89B8FD09, 0x8D79E0BE, 0x803AC667, 0x84FBDBD0, 0x9ABC8BD5, 0x9E7D9662, 0x933EB0BB, 0x97FFAD0C,
0xAFB010B1, 0xAB710D06, 0xA6322BDF, 0xA2F33668, 0xBCB4666D, 0xB8757BDA, 0xB5365D03, 0xB1F740B4,
}

View File

@@ -35,7 +35,7 @@ type MuxerData struct {
}
// parseData parses a payload spanning over multiple packets and returns a set of data
func parseData(ps []*Packet, prs PacketsParser, pm *programMap) (ds []*DemuxerData, err error) {
func parseData(ps []*Packet, prs PacketsParser, pm programMap) (ds []*DemuxerData, err error) {
// Use custom parser first
if prs != nil {
var skip bool
@@ -99,7 +99,7 @@ func parseData(ps []*Packet, prs PacketsParser, pm *programMap) (ds []*DemuxerDa
}
// isPSIPayload checks whether the payload is a PSI one
func isPSIPayload(pid uint16, pm *programMap) bool {
func isPSIPayload(pid uint16, pm programMap) bool {
return pid == PIDPAT || // PAT
pm.exists(pid) || // PMT
((pid >= 0x10 && pid <= 0x14) || (pid >= 0x1e && pid <= 0x1f)) //DVB

View File

@@ -145,7 +145,7 @@ func hasPESOptionalHeader(streamID uint8) bool {
return streamID != StreamIDPaddingStream && streamID != StreamIDPrivateStream2
}
// parsePESHeader parses a PES header
// parsePESData parses a PES header
func parsePESHeader(i *astikit.BytesIterator) (h *PESHeader, dataStart, dataEnd int, err error) {
// Create header
h = &PESHeader{}

View File

@@ -32,7 +32,6 @@ const (
StreamTypeAC3Audio StreamType = 0x81
StreamTypeDTSAudio StreamType = 0x82
StreamTypeTRUEHDAudio StreamType = 0x83
StreamTypeSCTE35 StreamType = 0x86
StreamTypeEAC3Audio StreamType = 0x87
)
@@ -232,8 +231,6 @@ func (t StreamType) String() string {
return "DTS Audio"
case StreamTypeTRUEHDAudio:
return "TRUEHD Audio"
case StreamTypeSCTE35:
return "SCTE 35"
case StreamTypeEAC3Audio:
return "EAC3 Audio"
}

View File

@@ -27,7 +27,7 @@ type Demuxer struct {
optPacketsParser PacketsParser
packetBuffer *packetBuffer
packetPool *packetPool
programMap *programMap
programMap programMap
r io.Reader
}
@@ -40,10 +40,10 @@ func NewDemuxer(ctx context.Context, r io.Reader, opts ...func(*Demuxer)) (d *De
// Init
d = &Demuxer{
ctx: ctx,
packetPool: newPacketPool(),
programMap: newProgramMap(),
r: r,
}
d.packetPool = newPacketPool(d.optPacketsParser, d.programMap)
// Apply options
for _, opt := range opts {
@@ -119,8 +119,7 @@ func (dmx *Demuxer) NextData() (d *DemuxerData, err error) {
}
// Parse data
var errParseData error
if ds, errParseData = parseData(ps, dmx.optPacketsParser, dmx.programMap); errParseData != nil {
if ds, err = parseData(ps, dmx.optPacketsParser, dmx.programMap); err != nil {
// We need to silence this error as there may be some incomplete data here
// We still want to try to parse all packets, in case final data is complete
continue
@@ -128,7 +127,6 @@ func (dmx *Demuxer) NextData() (d *DemuxerData, err error) {
// Update data
if d = dmx.updateData(ds); d != nil {
err = nil
return
}
}
@@ -182,7 +180,7 @@ func (dmx *Demuxer) updateData(ds []*DemuxerData) (d *DemuxerData) {
func (dmx *Demuxer) Rewind() (n int64, err error) {
dmx.dataBuffer = []*DemuxerData{}
dmx.packetBuffer = nil
dmx.packetPool = newPacketPool(dmx.optPacketsParser, dmx.programMap)
dmx.packetPool = newPacketPool()
if n, err = rewind(dmx.r); err != nil {
err = fmt.Errorf("astits: rewinding reader failed: %w", err)
return

View File

@@ -4,9 +4,8 @@ import (
"bytes"
"context"
"errors"
"io"
"github.com/asticode/go-astikit"
"io"
)
const (
@@ -29,15 +28,11 @@ type Muxer struct {
packetSize int
tablesRetransmitPeriod int // period in PES packets
pm *programMap // pid -> programNumber
pmUpdated bool
pm programMap // pid -> programNumber
pmt PMTData
pmtUpdated bool
nextPID uint16
patVersion wrappingCounter
pmtVersion wrappingCounter
patCC wrappingCounter
pmtCC wrappingCounter
patBytes bytes.Buffer
pmtBytes bytes.Buffer
@@ -87,9 +82,6 @@ func NewMuxer(ctx context.Context, w io.Writer, opts ...func(*Muxer)) *Muxer {
patVersion: newWrappingCounter(0b11111),
pmtVersion: newWrappingCounter(0b11111),
patCC: newWrappingCounter(0b1111),
pmtCC: newWrappingCounter(0b1111),
esContexts: map[uint16]*esContext{},
}
@@ -98,7 +90,6 @@ func NewMuxer(ctx context.Context, w io.Writer, opts ...func(*Muxer)) *Muxer {
// TODO multiple programs support
m.pm.set(pmtStartPID, programNumberStart)
m.pmUpdated = true
for _, opt := range opts {
opt(m)
@@ -128,7 +119,6 @@ func (m *Muxer) AddElementaryStream(es PMTElementaryStream) error {
m.esContexts[es.ElementaryPID] = newEsContext(&es)
// invalidate pmt cache
m.pmtBytes.Reset()
m.pmtUpdated = true
return nil
}
@@ -148,14 +138,12 @@ func (m *Muxer) RemoveElementaryStream(pid uint16) error {
m.pmt.ElementaryStreams = append(m.pmt.ElementaryStreams[:foundIdx], m.pmt.ElementaryStreams[foundIdx+1:]...)
delete(m.esContexts, pid)
m.pmtBytes.Reset()
m.pmtUpdated = true
return nil
}
// SetPCRPID marks pid as one to look PCRs in
func (m *Muxer) SetPCRPID(pid uint16) {
m.pmt.PCRPID = pid
m.pmtUpdated = true
}
// WriteData writes MuxerData to TS stream
@@ -187,7 +175,7 @@ func (m *Muxer) WriteData(d *MuxerData) (int, error) {
pktLen := 1 + mpegTsPacketHeaderSize // sync byte + header
pkt := Packet{
Header: &PacketHeader{
ContinuityCounter: uint8(ctx.cc.inc()),
ContinuityCounter: uint8(ctx.cc.get()),
HasAdaptationField: writeAf,
HasPayload: false,
PayloadUnitStartIndicator: false,
@@ -296,12 +284,16 @@ func (m *Muxer) retransmitTables(force bool) (int, error) {
func (m *Muxer) WriteTables() (int, error) {
bytesWritten := 0
if err := m.generatePAT(); err != nil {
return bytesWritten, err
if m.patBytes.Len() != m.packetSize {
if err := m.generatePAT(); err != nil {
return bytesWritten, err
}
}
if err := m.generatePMT(); err != nil {
return bytesWritten, err
if m.pmtBytes.Len() != m.packetSize {
if err := m.generatePMT(); err != nil {
return bytesWritten, err
}
}
n, err := m.w.Write(m.patBytes.Bytes())
@@ -321,12 +313,6 @@ func (m *Muxer) WriteTables() (int, error) {
func (m *Muxer) generatePAT() error {
d := m.pm.toPATData()
versionNumber := m.patVersion.get()
if m.pmUpdated {
versionNumber = m.patVersion.inc()
}
syntax := &PSISectionSyntax{
Data: &PSISectionSyntaxData{PAT: d},
Header: &PSISectionSyntaxHeader{
@@ -335,7 +321,7 @@ func (m *Muxer) generatePAT() error {
//LastSectionNumber: 0,
//SectionNumber: 0,
TableIDExtension: d.TransportStreamID,
VersionNumber: uint8(versionNumber),
VersionNumber: uint8(m.patVersion.get()),
},
}
section := PSISection{
@@ -364,7 +350,6 @@ func (m *Muxer) generatePAT() error {
HasPayload: true,
PayloadUnitStartIndicator: true,
PID: PIDPAT,
ContinuityCounter: uint8(m.patCC.inc()),
},
Payload: m.buf.Bytes(),
}
@@ -373,8 +358,6 @@ func (m *Muxer) generatePAT() error {
return err
}
m.pmUpdated = false
return nil
}
@@ -390,11 +373,6 @@ func (m *Muxer) generatePMT() error {
return ErrPCRPIDInvalid
}
versionNumber := m.pmtVersion.get()
if m.pmtUpdated {
versionNumber = m.pmtVersion.inc()
}
syntax := &PSISectionSyntax{
Data: &PSISectionSyntaxData{PMT: &m.pmt},
Header: &PSISectionSyntaxHeader{
@@ -403,7 +381,7 @@ func (m *Muxer) generatePMT() error {
//LastSectionNumber: 0,
//SectionNumber: 0,
TableIDExtension: m.pmt.ProgramNumber,
VersionNumber: uint8(versionNumber),
VersionNumber: uint8(m.pmtVersion.get()),
},
}
section := PSISection{
@@ -432,7 +410,6 @@ func (m *Muxer) generatePMT() error {
HasPayload: true,
PayloadUnitStartIndicator: true,
PID: pmtStartPID, // FIXME multiple programs support
ContinuityCounter: uint8(m.pmtCC.inc()),
},
Payload: m.buf.Bytes(),
}
@@ -441,7 +418,5 @@ func (m *Muxer) generatePMT() error {
return err
}
m.pmtUpdated = false
return nil
}

View File

@@ -5,76 +5,17 @@ import (
"sync"
)
// packetAccumulator keeps track of packets for a single PID and decides when to flush them
type packetAccumulator struct {
parser PacketsParser
pid uint16
programMap *programMap
q []*Packet
}
// newPacketAccumulator creates a new packet queue for a single PID
func newPacketAccumulator(pid uint16, parser PacketsParser, programMap *programMap) *packetAccumulator {
return &packetAccumulator{
parser: parser,
pid: pid,
programMap: programMap,
}
}
// add adds a new packet for this PID to the queue
func (b *packetAccumulator) add(p *Packet) (ps []*Packet) {
mps := b.q
// Empty buffer if we detect a discontinuity
if hasDiscontinuity(mps, p) {
mps = []*Packet{}
}
// Throw away packet if it's the same as the previous one
if isSameAsPrevious(mps, p) {
return
}
// Flush buffer if new payload starts here
if p.Header.PayloadUnitStartIndicator {
ps = mps
mps = []*Packet{p}
} else {
mps = append(mps, p)
}
// Check if PSI payload is complete
if b.programMap != nil &&
(b.pid == PIDPAT || b.programMap.exists(b.pid)) {
// TODO Use partial data parsing instead
if _, err := parseData(mps, b.parser, b.programMap); err == nil {
ps = mps
mps = nil
}
}
b.q = mps
return
}
// packetPool represents a queue of packets for each PID in the stream
// packetPool represents a pool of packets
type packetPool struct {
b map[uint16]*packetAccumulator // Indexed by PID
b map[uint16][]*Packet // Indexed by PID
m *sync.Mutex
parser PacketsParser
programMap *programMap
}
// newPacketPool creates a new packet pool with an optional parser and programMap
func newPacketPool(parser PacketsParser, programMap *programMap) *packetPool {
// newPacketPool creates a new packet pool
func newPacketPool() *packetPool {
return &packetPool{
b: make(map[uint16]*packetAccumulator),
b: make(map[uint16][]*Packet),
m: &sync.Mutex{},
parser: parser,
programMap: programMap,
}
}
@@ -95,13 +36,37 @@ func (b *packetPool) add(p *Packet) (ps []*Packet) {
b.m.Lock()
defer b.m.Unlock()
// Make sure accumulator exists
if _, ok := b.b[p.Header.PID]; !ok {
b.b[p.Header.PID] = newPacketAccumulator(p.Header.PID, b.parser, b.programMap)
// Init buffer
var mps []*Packet
var ok bool
if mps, ok = b.b[p.Header.PID]; !ok {
mps = []*Packet{}
}
// Add to the accumulator
return b.b[p.Header.PID].add(p)
// Empty buffer if we detect a discontinuity
if hasDiscontinuity(mps, p) {
mps = []*Packet{}
}
// Throw away packet if it's the same as the previous one
if isSameAsPrevious(mps, p) {
return
}
// Add packet
if len(mps) > 0 || (len(mps) == 0 && p.Header.PayloadUnitStartIndicator) {
mps = append(mps, p)
}
// Check payload unit start indicator
if p.Header.PayloadUnitStartIndicator && len(mps) > 1 {
ps = mps[:len(mps)-1]
mps = []*Packet{p}
}
// Assign
b.b[p.Header.PID] = mps
return
}
// dump dumps the packet pool by looking for the first item with packets inside
@@ -114,7 +79,7 @@ func (b *packetPool) dump() (ps []*Packet) {
}
sort.Ints(keys)
for _, k := range keys {
ps = b.b[uint16(k)].q
ps = b.b[uint16(k)]
delete(b.b, uint16(k))
if len(ps) > 0 {
return

View File

@@ -9,8 +9,8 @@ type programMap struct {
}
// newProgramMap creates a new program ids map
func newProgramMap() *programMap {
return &programMap{
func newProgramMap() programMap {
return programMap{
m: &sync.Mutex{},
p: make(map[uint16]uint16),
}

View File

@@ -1,25 +1,22 @@
package astits
type wrappingCounter struct {
value int
wrapAt int
value int
}
func newWrappingCounter(wrapAt int) wrappingCounter {
return wrappingCounter{
value: wrapAt + 1,
wrapAt: wrapAt,
}
}
// returns current counter state and increments internal value
func (c *wrappingCounter) get() int {
return c.value
}
func (c *wrappingCounter) inc() int {
ret := c.value
c.value++
if c.value > c.wrapAt {
c.value = 0
}
return c.value
return ret
}