Skip to content

Commit

Permalink
improve code quality
Browse files Browse the repository at this point in the history
  • Loading branch information
zhenghaoz committed May 30, 2021
1 parent c63daab commit 4d1f0b2
Show file tree
Hide file tree
Showing 13 changed files with 129 additions and 59 deletions.
6 changes: 6 additions & 0 deletions base/filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@ type weightedItem struct {
weight float32
}

// TopKFilter filters out top k items with maximum weights.
type TopKFilter struct {
items []weightedItem
k int
}

// NewTopKFilter creates a top k filter.
func NewTopKFilter(k int) *TopKFilter {
filter := new(TopKFilter)
filter.items = make([]weightedItem, 0, k+1)
Expand Down Expand Up @@ -96,6 +98,7 @@ func (filter *TopKFilter) down(i0, n int) bool {
return i > i0
}

// PopAll pops all items in the filter with decreasing order.
func (filter *TopKFilter) PopAll() ([]int, []float32) {
items := make([]int, filter.Len())
weights := make([]float32, filter.Len())
Expand All @@ -110,11 +113,13 @@ type weightedString struct {
weight float32
}

// TopKStringFilter filters out top k strings with maximum weights.
type TopKStringFilter struct {
items []weightedString
k int
}

// NewTopKStringFilter creates a top k string filter.
func NewTopKStringFilter(k int) *TopKStringFilter {
filter := new(TopKStringFilter)
filter.items = make([]weightedString, 0, k+1)
Expand Down Expand Up @@ -187,6 +192,7 @@ func (filter *TopKStringFilter) down(i0, n int) bool {
return i > i0
}

// PopAll pops all strings in the filter with decreasing order.
func (filter *TopKStringFilter) PopAll() ([]string, []float32) {
items := make([]string, filter.Len())
weights := make([]float32, filter.Len())
Expand Down
10 changes: 10 additions & 0 deletions base/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"strconv"
)

// Index keeps the mapping between names (string) and indices (integer).
type Index interface {
Len() int
Add(name string)
Expand Down Expand Up @@ -79,22 +80,28 @@ func (idx *MapIndex) ToName(index int) string {
return idx.Names[index]
}

// GetNames returns all names in current index.
func (idx *MapIndex) GetNames() []string {
return idx.Names
}

// DirectIndex means that the name and its index is the same. For example,
// the index of "1" is 1, vice versa.
type DirectIndex struct {
Limit int
}

// NewDirectIndex create a direct mapping index.
func NewDirectIndex() *DirectIndex {
return &DirectIndex{Limit: 0}
}

// Len returns the number of names in current index.
func (idx *DirectIndex) Len() int {
return idx.Limit
}

// Add a name to current index.
func (idx *DirectIndex) Add(s string) {
i, err := strconv.Atoi(s)
if err != nil {
Expand All @@ -105,6 +112,7 @@ func (idx *DirectIndex) Add(s string) {
}
}

// ToNumber converts a name to corresponding index.
func (idx *DirectIndex) ToNumber(name string) int {
i, err := strconv.Atoi(name)
if err != nil {
Expand All @@ -116,13 +124,15 @@ func (idx *DirectIndex) ToNumber(name string) int {
return i
}

// ToName converts a index to corresponding name.
func (idx *DirectIndex) ToName(index int) string {
if index >= idx.Limit {
panic("index out of range")
}
return strconv.Itoa(index)
}

// GetNames returns all names in current index.
func (idx *DirectIndex) GetNames() []string {
names := make([]string, idx.Limit)
for i := range names {
Expand Down
84 changes: 42 additions & 42 deletions base/parallel.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,54 +86,54 @@ type batchJob struct {
endId int
}

// BatchParallel run parallel jobs in batches to reduce the cost of context switch.
func BatchParallel(nJobs int, nWorkers int, batchSize int, worker func(workerId, beginJobId, endJobId int) error) error {
if nWorkers == 1 {
return worker(0, 0, nJobs)
} else {
const chanSize = 64
const chanEOF = -1
c := make(chan batchJob, chanSize)
// producer
go func() {
}
const chanSize = 64
const chanEOF = -1
c := make(chan batchJob, chanSize)
// producer
go func() {
defer CheckPanic()
// send jobs
for i := 0; i < nJobs; i += batchSize {
c <- batchJob{beginId: i, endId: Min(i+batchSize, nJobs)}
}
// send EOF
for i := 0; i < nWorkers; i++ {
c <- batchJob{beginId: chanEOF, endId: chanEOF}
}
}()
// consumer
var wg sync.WaitGroup
wg.Add(nWorkers)
errs := make([]error, nJobs)
for j := 0; j < nWorkers; j++ {
// start workers
go func(workerId int) {
defer CheckPanic()
// send jobs
for i := 0; i < nJobs; i += batchSize {
c <- batchJob{beginId: i, endId: Min(i+batchSize, nJobs)}
}
// send EOF
for i := 0; i < nWorkers; i++ {
c <- batchJob{beginId: chanEOF, endId: chanEOF}
}
}()
// consumer
var wg sync.WaitGroup
wg.Add(nWorkers)
errs := make([]error, nJobs)
for j := 0; j < nWorkers; j++ {
// start workers
go func(workerId int) {
defer CheckPanic()
defer wg.Done()
for {
// read job
job := <-c
if job.beginId == chanEOF {
return
}
// run job
if err := worker(workerId, job.beginId, job.endId); err != nil {
errs[job.beginId] = err
return
}
defer wg.Done()
for {
// read job
job := <-c
if job.beginId == chanEOF {
return
}
// run job
if err := worker(workerId, job.beginId, job.endId); err != nil {
errs[job.beginId] = err
return
}
}(j)
}
wg.Wait()
// check errors
for _, err := range errs {
if err != nil {
return err
}
}(j)
}
wg.Wait()
// check errors
for _, err := range errs {
if err != nil {
return err
}
}
return nil
Expand Down
3 changes: 2 additions & 1 deletion base/random.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func (rng RandomGenerator) UniformMatrix(row, col int, low, high float32) [][]fl
return ret
}

// NewNormalVector makes a vec filled with normal random floats.
// NormalVector64 makes a vec filled with normal random floats.
func (rng RandomGenerator) NormalVector64(size int, mean, stdDev float64) []float64 {
ret := make([]float64, size)
for i := 0; i < len(ret); i++ {
Expand All @@ -84,6 +84,7 @@ func (rng RandomGenerator) NormalMatrix64(row, col int, mean, stdDev float64) []
return ret
}

// Sample n values between low and high, but not in exclude.
func (rng RandomGenerator) Sample(low, high, n int, exclude ...*iset.Set) []int {
intervalLength := high - low
excludeSet := iset.Union(exclude...)
Expand Down
10 changes: 10 additions & 0 deletions base/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,17 @@ func init() {
SetProductionLogger()
}

// Logger get current logger
func Logger() *zap.Logger {
return logger
}

// SetProductionLogger set current logger in production mode.
func SetProductionLogger() {
logger, _ = zap.NewProduction()
}

// SetDevelopmentLogger set current logger in development mode.
func SetDevelopmentLogger() {
logger, _ = zap.NewDevelopment()
}
Expand All @@ -52,6 +55,7 @@ func Max(a ...int) int {
return maximum
}

// Min finds the minimum in a vector of integers. Panic if the slice is empty.
func Min(a ...int) int {
if len(a) == 0 {
panic("can't get the minimum from empty vec")
Expand All @@ -65,6 +69,7 @@ func Min(a ...int) int {
return minimum
}

// RangeInt generate a slice [0, ..., n-1].
func RangeInt(n int) []int {
a := make([]int, n)
for i := range a {
Expand All @@ -73,6 +78,7 @@ func RangeInt(n int) []int {
return a
}

// NewMatrix32 creates a 2D matrix of 32-bit floats.
func NewMatrix32(row, col int) [][]float32 {
ret := make([][]float32, row)
for i := range ret {
Expand All @@ -81,6 +87,7 @@ func NewMatrix32(row, col int) [][]float32 {
return ret
}

// NewMatrixInt creates a 2D matrix of integers.
func NewMatrixInt(row, col int) [][]int {
ret := make([][]int, row)
for i := range ret {
Expand All @@ -89,16 +96,19 @@ func NewMatrixInt(row, col int) [][]int {
return ret
}

// Now returns the current time in the format of `2006-01-02T15:04:05Z07:00`.
func Now() string {
return time.Now().Format("2006-01-02T15:04:05Z07:00")
}

// CheckPanic catches panic.
func CheckPanic() {
if r := recover(); r != nil {
Logger().Error("panic recovered", zap.Any("panic", r))
}
}

// Hex returns the hex form of a 64-bit integer.
func Hex(v int64) string {
return fmt.Sprintf("%x", v)
}
2 changes: 2 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ type Config struct {
Recommend RecommendConfig `toml:"recommend"`
}

// LoadDefaultIfNil loads default settings if config is nil.
func (config *Config) LoadDefaultIfNil() *Config {
if config == nil {
return &Config{
Expand Down Expand Up @@ -89,6 +90,7 @@ func (config *MasterConfig) LoadDefaultIfNil() *MasterConfig {
return config
}

// RecommendConfig is the configuration of recommendation setup.
type RecommendConfig struct {
PopularWindow int `toml:"popular_window"`
FitPeriod int `toml:"fit_period"`
Expand Down
7 changes: 7 additions & 0 deletions floats/floats.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,14 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package floats

import (
"github.com/chewxy/math32"
)

// MatZero fills zeros in a matrix of 32-bit floats.
func MatZero(x [][]float32) {
for i := range x {
for j := range x[i] {
Expand All @@ -25,6 +27,7 @@ func MatZero(x [][]float32) {
}
}

// Zero fills zeros in a slice of 32-bit floats.
func Zero(a []float32) {
for i := range a {
a[i] = 0
Expand Down Expand Up @@ -129,6 +132,7 @@ func Dot(a, b []float32) (ret float32) {
return
}

// Min element of a slice of 32-bit floats.
func Min(x []float32) float32 {
if len(x) == 0 {
panic("floats: zero slice length")
Expand All @@ -142,6 +146,7 @@ func Min(x []float32) float32 {
return min
}

// Max element of a slice of 32-bit floats.
func Max(x []float32) float32 {
if len(x) == 0 {
panic("floats: zero slice length")
Expand All @@ -155,6 +160,7 @@ func Max(x []float32) float32 {
return max
}

// Sum of a slice of 32-bit floats.
func Sum(x []float32) float32 {
sum := float32(0)
for _, v := range x {
Expand All @@ -163,6 +169,7 @@ func Sum(x []float32) float32 {
return sum
}

// Mean of a slice of 32-bit floats.
func Mean(x []float32) float32 {
return Sum(x) / float32(len(x))
}
Expand Down
2 changes: 2 additions & 0 deletions master/rest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ func TestMaster_ImportFeedback(t *testing.T) {
"1\t4\tshare\t0001-01-01 00:00:00 +0000 UTC\n"))
assert.Nil(t, err)
err = writer.Close()
assert.Nil(t, err)
req := httptest.NewRequest("POST", "https://example.com/", buf)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
Expand Down Expand Up @@ -247,6 +248,7 @@ func TestMaster_ImportFeedback_Default(t *testing.T) {
"share,1,4,0001-01-01 00:00:00 +0000 UTC\r\n"))
assert.Nil(t, err)
err = writer.Close()
assert.Nil(t, err)
req := httptest.NewRequest("POST", "https://example.com/", buf)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
Expand Down
Loading

0 comments on commit 4d1f0b2

Please sign in to comment.