Commit 436392fc by zhaoyanchao

重新添加文件

parent e334d0d4
# This is the official list of Go-MySQL-Driver authors for copyright purposes.
# If you are submitting a patch, please add your name or the name of the
# organization which holds the copyright to this list in alphabetical order.
# Names should be added to this file as
# Name <email address>
# The email address is not required for organizations.
# Please keep the list sorted.
# Individual Persons
Aaron Hopkins <go-sql-driver at die.net>
Achille Roussel <achille.roussel at gmail.com>
Alex Snast <alexsn at fb.com>
Alexey Palazhchenko <alexey.palazhchenko at gmail.com>
Andrew Reid <andrew.reid at tixtrack.com>
Animesh Ray <mail.rayanimesh at gmail.com>
Arne Hormann <arnehormann at gmail.com>
Ariel Mashraki <ariel at mashraki.co.il>
Asta Xie <xiemengjun at gmail.com>
Bulat Gaifullin <gaifullinbf at gmail.com>
Caine Jette <jette at alum.mit.edu>
Carlos Nieto <jose.carlos at menteslibres.net>
Chris Moos <chris at tech9computers.com>
Craig Wilson <craiggwilson at gmail.com>
Daniel Montoya <dsmontoyam at gmail.com>
Daniel Nichter <nil at codenode.com>
Daniël van Eeden <git at myname.nl>
Dave Protasowski <dprotaso at gmail.com>
DisposaBoy <disposaboy at dby.me>
Egor Smolyakov <egorsmkv at gmail.com>
Erwan Martin <hello at erwan.io>
Evan Shaw <evan at vendhq.com>
Frederick Mayle <frederickmayle at gmail.com>
Gustavo Kristic <gkristic at gmail.com>
Hajime Nakagami <nakagami at gmail.com>
Hanno Braun <mail at hannobraun.com>
Henri Yandell <flamefew at gmail.com>
Hirotaka Yamamoto <ymmt2005 at gmail.com>
Huyiguang <hyg at webterren.com>
ICHINOSE Shogo <shogo82148 at gmail.com>
Ilia Cimpoes <ichimpoesh at gmail.com>
INADA Naoki <songofacandy at gmail.com>
Jacek Szwec <szwec.jacek at gmail.com>
James Harr <james.harr at gmail.com>
Jeff Hodges <jeff at somethingsimilar.com>
Jeffrey Charles <jeffreycharles at gmail.com>
Jerome Meyer <jxmeyer at gmail.com>
Jiajia Zhong <zhong2plus at gmail.com>
Jian Zhen <zhenjl at gmail.com>
Joshua Prunier <joshua.prunier at gmail.com>
Julien Lefevre <julien.lefevr at gmail.com>
Julien Schmidt <go-sql-driver at julienschmidt.com>
Justin Li <jli at j-li.net>
Justin Nuß <nuss.justin at gmail.com>
Kamil Dziedzic <kamil at klecza.pl>
Kei Kamikawa <x00.x7f.x86 at gmail.com>
Kevin Malachowski <kevin at chowski.com>
Kieron Woodhouse <kieron.woodhouse at infosum.com>
Lennart Rudolph <lrudolph at hmc.edu>
Leonardo YongUk Kim <dalinaum at gmail.com>
Linh Tran Tuan <linhduonggnu at gmail.com>
Lion Yang <lion at aosc.xyz>
Luca Looz <luca.looz92 at gmail.com>
Lucas Liu <extrafliu at gmail.com>
Luke Scott <luke at webconnex.com>
Maciej Zimnoch <maciej.zimnoch at codilime.com>
Michael Woolnough <michael.woolnough at gmail.com>
Nathanial Murphy <nathanial.murphy at gmail.com>
Nicola Peduzzi <thenikso at gmail.com>
Olivier Mengué <dolmen at cpan.org>
oscarzhao <oscarzhaosl at gmail.com>
Paul Bonser <misterpib at gmail.com>
Peter Schultz <peter.schultz at classmarkets.com>
Rebecca Chin <rchin at pivotal.io>
Reed Allman <rdallman10 at gmail.com>
Richard Wilkes <wilkes at me.com>
Robert Russell <robert at rrbrussell.com>
Runrioter Wung <runrioter at gmail.com>
Sho Iizuka <sho.i518 at gmail.com>
Sho Ikeda <suicaicoca at gmail.com>
Shuode Li <elemount at qq.com>
Simon J Mudd <sjmudd at pobox.com>
Soroush Pour <me at soroushjp.com>
Stan Putrya <root.vagner at gmail.com>
Stanley Gunawan <gunawan.stanley at gmail.com>
Steven Hartland <steven.hartland at multiplay.co.uk>
Tan Jinhua <312841925 at qq.com>
Thomas Wodarek <wodarekwebpage at gmail.com>
Tim Ruffles <timruffles at gmail.com>
Tom Jenkinson <tom at tjenkinson.me>
Vladimir Kovpak <cn007b at gmail.com>
Vladyslav Zhelezniak <zhvladi at gmail.com>
Xiangyu Hu <xiangyu.hu at outlook.com>
Xiaobing Jiang <s7v7nislands at gmail.com>
Xiuming Chen <cc at cxm.cc>
Xuehong Chan <chanxuehong at gmail.com>
Zhenye Xie <xiezhenye at gmail.com>
Zhixin Wen <john.wenzhixin at gmail.com>
# Organizations
Barracuda Networks, Inc.
Counting Ltd.
DigitalOcean Inc.
Facebook Inc.
GitHub Inc.
Google Inc.
InfoSum Ltd.
Keybase Inc.
Multiplay Ltd.
Percona LLC
Pivotal Inc.
Stripe Inc.
Zendesk Inc.
## Version 1.5 (2020-01-07)
Changes:
- Dropped support Go 1.9 and lower (#823, #829, #886, #1016, #1017)
- Improve buffer handling (#890)
- Document potentially insecure TLS configs (#901)
- Use a double-buffering scheme to prevent data races (#943)
- Pass uint64 values without converting them to string (#838, #955)
- Update collations and make utf8mb4 default (#877, #1054)
- Make NullTime compatible with sql.NullTime in Go 1.13+ (#995)
- Removed CloudSQL support (#993, #1007)
- Add Go Module support (#1003)
New Features:
- Implement support of optional TLS (#900)
- Check connection liveness (#934, #964, #997, #1048, #1051, #1052)
- Implement Connector Interface (#941, #958, #1020, #1035)
Bugfixes:
- Mark connections as bad on error during ping (#875)
- Mark connections as bad on error during dial (#867)
- Fix connection leak caused by rapid context cancellation (#1024)
- Mark connections as bad on error during Conn.Prepare (#1030)
## Version 1.4.1 (2018-11-14)
Bugfixes:
- Fix TIME format for binary columns (#818)
- Fix handling of empty auth plugin names (#835)
- Fix caching_sha2_password with empty password (#826)
- Fix canceled context broke mysqlConn (#862)
- Fix OldAuthSwitchRequest support (#870)
- Fix Auth Response packet for cleartext password (#887)
## Version 1.4 (2018-06-03)
Changes:
- Documentation fixes (#530, #535, #567)
- Refactoring (#575, #579, #580, #581, #603, #615, #704)
- Cache column names (#444)
- Sort the DSN parameters in DSNs generated from a config (#637)
- Allow native password authentication by default (#644)
- Use the default port if it is missing in the DSN (#668)
- Removed the `strict` mode (#676)
- Do not query `max_allowed_packet` by default (#680)
- Dropped support Go 1.6 and lower (#696)
- Updated `ConvertValue()` to match the database/sql/driver implementation (#760)
- Document the usage of `0000-00-00T00:00:00` as the time.Time zero value (#783)
- Improved the compatibility of the authentication system (#807)
New Features:
- Multi-Results support (#537)
- `rejectReadOnly` DSN option (#604)
- `context.Context` support (#608, #612, #627, #761)
- Transaction isolation level support (#619, #744)
- Read-Only transactions support (#618, #634)
- `NewConfig` function which initializes a config with default values (#679)
- Implemented the `ColumnType` interfaces (#667, #724)
- Support for custom string types in `ConvertValue` (#623)
- Implemented `NamedValueChecker`, improving support for uint64 with high bit set (#690, #709, #710)
- `caching_sha2_password` authentication plugin support (#794, #800, #801, #802)
- Implemented `driver.SessionResetter` (#779)
- `sha256_password` authentication plugin support (#808)
Bugfixes:
- Use the DSN hostname as TLS default ServerName if `tls=true` (#564, #718)
- Fixed LOAD LOCAL DATA INFILE for empty files (#590)
- Removed columns definition cache since it sometimes cached invalid data (#592)
- Don't mutate registered TLS configs (#600)
- Make RegisterTLSConfig concurrency-safe (#613)
- Handle missing auth data in the handshake packet correctly (#646)
- Do not retry queries when data was written to avoid data corruption (#302, #736)
- Cache the connection pointer for error handling before invalidating it (#678)
- Fixed imports for appengine/cloudsql (#700)
- Fix sending STMT_LONG_DATA for 0 byte data (#734)
- Set correct capacity for []bytes read from length-encoded strings (#766)
- Make RegisterDial concurrency-safe (#773)
## Version 1.3 (2016-12-01)
Changes:
- Go 1.1 is no longer supported
- Use decimals fields in MySQL to format time types (#249)
- Buffer optimizations (#269)
- TLS ServerName defaults to the host (#283)
- Refactoring (#400, #410, #437)
- Adjusted documentation for second generation CloudSQL (#485)
- Documented DSN system var quoting rules (#502)
- Made statement.Close() calls idempotent to avoid errors in Go 1.6+ (#512)
New Features:
- Enable microsecond resolution on TIME, DATETIME and TIMESTAMP (#249)
- Support for returning table alias on Columns() (#289, #359, #382)
- Placeholder interpolation, can be actived with the DSN parameter `interpolateParams=true` (#309, #318, #490)
- Support for uint64 parameters with high bit set (#332, #345)
- Cleartext authentication plugin support (#327)
- Exported ParseDSN function and the Config struct (#403, #419, #429)
- Read / Write timeouts (#401)
- Support for JSON field type (#414)
- Support for multi-statements and multi-results (#411, #431)
- DSN parameter to set the driver-side max_allowed_packet value manually (#489)
- Native password authentication plugin support (#494, #524)
Bugfixes:
- Fixed handling of queries without columns and rows (#255)
- Fixed a panic when SetKeepAlive() failed (#298)
- Handle ERR packets while reading rows (#321)
- Fixed reading NULL length-encoded integers in MySQL 5.6+ (#349)
- Fixed absolute paths support in LOAD LOCAL DATA INFILE (#356)
- Actually zero out bytes in handshake response (#378)
- Fixed race condition in registering LOAD DATA INFILE handler (#383)
- Fixed tests with MySQL 5.7.9+ (#380)
- QueryUnescape TLS config names (#397)
- Fixed "broken pipe" error by writing to closed socket (#390)
- Fixed LOAD LOCAL DATA INFILE buffering (#424)
- Fixed parsing of floats into float64 when placeholders are used (#434)
- Fixed DSN tests with Go 1.7+ (#459)
- Handle ERR packets while waiting for EOF (#473)
- Invalidate connection on error while discarding additional results (#513)
- Allow terminating packets of length 0 (#516)
## Version 1.2 (2014-06-03)
Changes:
- We switched back to a "rolling release". `go get` installs the current master branch again
- Version v1 of the driver will not be maintained anymore. Go 1.0 is no longer supported by this driver
- Exported errors to allow easy checking from application code
- Enabled TCP Keepalives on TCP connections
- Optimized INFILE handling (better buffer size calculation, lazy init, ...)
- The DSN parser also checks for a missing separating slash
- Faster binary date / datetime to string formatting
- Also exported the MySQLWarning type
- mysqlConn.Close returns the first error encountered instead of ignoring all errors
- writePacket() automatically writes the packet size to the header
- readPacket() uses an iterative approach instead of the recursive approach to merge splitted packets
New Features:
- `RegisterDial` allows the usage of a custom dial function to establish the network connection
- Setting the connection collation is possible with the `collation` DSN parameter. This parameter should be preferred over the `charset` parameter
- Logging of critical errors is configurable with `SetLogger`
- Google CloudSQL support
Bugfixes:
- Allow more than 32 parameters in prepared statements
- Various old_password fixes
- Fixed TestConcurrent test to pass Go's race detection
- Fixed appendLengthEncodedInteger for large numbers
- Renamed readLengthEnodedString to readLengthEncodedString and skipLengthEnodedString to skipLengthEncodedString (fixed typo)
## Version 1.1 (2013-11-02)
Changes:
- Go-MySQL-Driver now requires Go 1.1
- Connections now use the collation `utf8_general_ci` by default. Adding `&charset=UTF8` to the DSN should not be necessary anymore
- Made closing rows and connections error tolerant. This allows for example deferring rows.Close() without checking for errors
- `[]byte(nil)` is now treated as a NULL value. Before, it was treated like an empty string / `[]byte("")`
- DSN parameter values must now be url.QueryEscape'ed. This allows text values to contain special characters, such as '&'.
- Use the IO buffer also for writing. This results in zero allocations (by the driver) for most queries
- Optimized the buffer for reading
- stmt.Query now caches column metadata
- New Logo
- Changed the copyright header to include all contributors
- Improved the LOAD INFILE documentation
- The driver struct is now exported to make the driver directly accessible
- Refactored the driver tests
- Added more benchmarks and moved all to a separate file
- Other small refactoring
New Features:
- Added *old_passwords* support: Required in some cases, but must be enabled by adding `allowOldPasswords=true` to the DSN since it is insecure
- Added a `clientFoundRows` parameter: Return the number of matching rows instead of the number of rows changed on UPDATEs
- Added TLS/SSL support: Use a TLS/SSL encrypted connection to the server. Custom TLS configs can be registered and used
Bugfixes:
- Fixed MySQL 4.1 support: MySQL 4.1 sends packets with lengths which differ from the specification
- Convert to DB timezone when inserting `time.Time`
- Splitted packets (more than 16MB) are now merged correctly
- Fixed false positive `io.EOF` errors when the data was fully read
- Avoid panics on reuse of closed connections
- Fixed empty string producing false nil values
- Fixed sign byte for positive TIME fields
## Version 1.0 (2013-05-14)
Initial Release
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"bytes"
"context"
"database/sql"
"database/sql/driver"
"fmt"
"math"
"runtime"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
type TB testing.B
func (tb *TB) check(err error) {
if err != nil {
tb.Fatal(err)
}
}
func (tb *TB) checkDB(db *sql.DB, err error) *sql.DB {
tb.check(err)
return db
}
func (tb *TB) checkRows(rows *sql.Rows, err error) *sql.Rows {
tb.check(err)
return rows
}
func (tb *TB) checkStmt(stmt *sql.Stmt, err error) *sql.Stmt {
tb.check(err)
return stmt
}
func initDB(b *testing.B, queries ...string) *sql.DB {
tb := (*TB)(b)
db := tb.checkDB(sql.Open("mysql", dsn))
for _, query := range queries {
if _, err := db.Exec(query); err != nil {
b.Fatalf("error on %q: %v", query, err)
}
}
return db
}
const concurrencyLevel = 10
func BenchmarkQuery(b *testing.B) {
tb := (*TB)(b)
b.StopTimer()
b.ReportAllocs()
db := initDB(b,
"DROP TABLE IF EXISTS foo",
"CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))",
`INSERT INTO foo VALUES (1, "one")`,
`INSERT INTO foo VALUES (2, "two")`,
)
db.SetMaxIdleConns(concurrencyLevel)
defer db.Close()
stmt := tb.checkStmt(db.Prepare("SELECT val FROM foo WHERE id=?"))
defer stmt.Close()
remain := int64(b.N)
var wg sync.WaitGroup
wg.Add(concurrencyLevel)
defer wg.Wait()
b.StartTimer()
for i := 0; i < concurrencyLevel; i++ {
go func() {
for {
if atomic.AddInt64(&remain, -1) < 0 {
wg.Done()
return
}
var got string
tb.check(stmt.QueryRow(1).Scan(&got))
if got != "one" {
b.Errorf("query = %q; want one", got)
wg.Done()
return
}
}
}()
}
}
func BenchmarkExec(b *testing.B) {
tb := (*TB)(b)
b.StopTimer()
b.ReportAllocs()
db := tb.checkDB(sql.Open("mysql", dsn))
db.SetMaxIdleConns(concurrencyLevel)
defer db.Close()
stmt := tb.checkStmt(db.Prepare("DO 1"))
defer stmt.Close()
remain := int64(b.N)
var wg sync.WaitGroup
wg.Add(concurrencyLevel)
defer wg.Wait()
b.StartTimer()
for i := 0; i < concurrencyLevel; i++ {
go func() {
for {
if atomic.AddInt64(&remain, -1) < 0 {
wg.Done()
return
}
if _, err := stmt.Exec(); err != nil {
b.Logf("stmt.Exec failed: %v", err)
b.Fail()
}
}
}()
}
}
// data, but no db writes
var roundtripSample []byte
func initRoundtripBenchmarks() ([]byte, int, int) {
if roundtripSample == nil {
roundtripSample = []byte(strings.Repeat("0123456789abcdef", 1024*1024))
}
return roundtripSample, 16, len(roundtripSample)
}
func BenchmarkRoundtripTxt(b *testing.B) {
b.StopTimer()
sample, min, max := initRoundtripBenchmarks()
sampleString := string(sample)
b.ReportAllocs()
tb := (*TB)(b)
db := tb.checkDB(sql.Open("mysql", dsn))
defer db.Close()
b.StartTimer()
var result string
for i := 0; i < b.N; i++ {
length := min + i
if length > max {
length = max
}
test := sampleString[0:length]
rows := tb.checkRows(db.Query(`SELECT "` + test + `"`))
if !rows.Next() {
rows.Close()
b.Fatalf("crashed")
}
err := rows.Scan(&result)
if err != nil {
rows.Close()
b.Fatalf("crashed")
}
if result != test {
rows.Close()
b.Errorf("mismatch")
}
rows.Close()
}
}
func BenchmarkRoundtripBin(b *testing.B) {
b.StopTimer()
sample, min, max := initRoundtripBenchmarks()
b.ReportAllocs()
tb := (*TB)(b)
db := tb.checkDB(sql.Open("mysql", dsn))
defer db.Close()
stmt := tb.checkStmt(db.Prepare("SELECT ?"))
defer stmt.Close()
b.StartTimer()
var result sql.RawBytes
for i := 0; i < b.N; i++ {
length := min + i
if length > max {
length = max
}
test := sample[0:length]
rows := tb.checkRows(stmt.Query(test))
if !rows.Next() {
rows.Close()
b.Fatalf("crashed")
}
err := rows.Scan(&result)
if err != nil {
rows.Close()
b.Fatalf("crashed")
}
if !bytes.Equal(result, test) {
rows.Close()
b.Errorf("mismatch")
}
rows.Close()
}
}
func BenchmarkInterpolation(b *testing.B) {
mc := &mysqlConn{
cfg: &Config{
InterpolateParams: true,
Loc: time.UTC,
},
maxAllowedPacket: maxPacketSize,
maxWriteSize: maxPacketSize - 1,
buf: newBuffer(nil),
}
args := []driver.Value{
int64(42424242),
float64(math.Pi),
false,
time.Unix(1423411542, 807015000),
[]byte("bytes containing special chars ' \" \a \x00"),
"string containing special chars ' \" \a \x00",
}
q := "SELECT ?, ?, ?, ?, ?, ?"
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := mc.interpolateParams(q, args)
if err != nil {
b.Fatal(err)
}
}
}
func benchmarkQueryContext(b *testing.B, db *sql.DB, p int) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
db.SetMaxIdleConns(p * runtime.GOMAXPROCS(0))
tb := (*TB)(b)
stmt := tb.checkStmt(db.PrepareContext(ctx, "SELECT val FROM foo WHERE id=?"))
defer stmt.Close()
b.SetParallelism(p)
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
var got string
for pb.Next() {
tb.check(stmt.QueryRow(1).Scan(&got))
if got != "one" {
b.Fatalf("query = %q; want one", got)
}
}
})
}
func BenchmarkQueryContext(b *testing.B) {
db := initDB(b,
"DROP TABLE IF EXISTS foo",
"CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))",
`INSERT INTO foo VALUES (1, "one")`,
`INSERT INTO foo VALUES (2, "two")`,
)
defer db.Close()
for _, p := range []int{1, 2, 3, 4} {
b.Run(fmt.Sprintf("%d", p), func(b *testing.B) {
benchmarkQueryContext(b, db, p)
})
}
}
func benchmarkExecContext(b *testing.B, db *sql.DB, p int) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
db.SetMaxIdleConns(p * runtime.GOMAXPROCS(0))
tb := (*TB)(b)
stmt := tb.checkStmt(db.PrepareContext(ctx, "DO 1"))
defer stmt.Close()
b.SetParallelism(p)
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if _, err := stmt.ExecContext(ctx); err != nil {
b.Fatal(err)
}
}
})
}
func BenchmarkExecContext(b *testing.B) {
db := initDB(b,
"DROP TABLE IF EXISTS foo",
"CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))",
`INSERT INTO foo VALUES (1, "one")`,
`INSERT INTO foo VALUES (2, "two")`,
)
defer db.Close()
for _, p := range []int{1, 2, 3, 4} {
b.Run(fmt.Sprintf("%d", p), func(b *testing.B) {
benchmarkQueryContext(b, db, p)
})
}
}
// BenchmarkQueryRawBytes benchmarks fetching 100 blobs using sql.RawBytes.
// "size=" means size of each blobs.
func BenchmarkQueryRawBytes(b *testing.B) {
var sizes []int = []int{100, 1000, 2000, 4000, 8000, 12000, 16000, 32000, 64000, 256000}
db := initDB(b,
"DROP TABLE IF EXISTS bench_rawbytes",
"CREATE TABLE bench_rawbytes (id INT PRIMARY KEY, val LONGBLOB)",
)
defer db.Close()
blob := make([]byte, sizes[len(sizes)-1])
for i := range blob {
blob[i] = 42
}
for i := 0; i < 100; i++ {
_, err := db.Exec("INSERT INTO bench_rawbytes VALUES (?, ?)", i, blob)
if err != nil {
b.Fatal(err)
}
}
for _, s := range sizes {
b.Run(fmt.Sprintf("size=%v", s), func(b *testing.B) {
db.SetMaxIdleConns(0)
db.SetMaxIdleConns(1)
b.ReportAllocs()
b.ResetTimer()
for j := 0; j < b.N; j++ {
rows, err := db.Query("SELECT LEFT(val, ?) as v FROM bench_rawbytes", s)
if err != nil {
b.Fatal(err)
}
nrows := 0
for rows.Next() {
var buf sql.RawBytes
err := rows.Scan(&buf)
if err != nil {
b.Fatal(err)
}
if len(buf) != s {
b.Fatalf("size mismatch: expected %v, got %v", s, len(buf))
}
nrows++
}
rows.Close()
if nrows != 100 {
b.Fatalf("numbers of rows mismatch: expected %v, got %v", 100, nrows)
}
}
})
}
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"io"
"net"
"time"
)
const defaultBufSize = 4096
const maxCachedBufSize = 256 * 1024
// A buffer which is used for both reading and writing.
// This is possible since communication on each connection is synchronous.
// In other words, we can't write and read simultaneously on the same connection.
// The buffer is similar to bufio.Reader / Writer but zero-copy-ish
// Also highly optimized for this particular use case.
// This buffer is backed by two byte slices in a double-buffering scheme
type buffer struct {
buf []byte // buf is a byte buffer who's length and capacity are equal.
nc net.Conn
idx int
length int
timeout time.Duration
dbuf [2][]byte // dbuf is an array with the two byte slices that back this buffer
flipcnt uint // flipccnt is the current buffer counter for double-buffering
}
// newBuffer allocates and returns a new buffer.
func newBuffer(nc net.Conn) buffer {
fg := make([]byte, defaultBufSize)
return buffer{
buf: fg,
nc: nc,
dbuf: [2][]byte{fg, nil},
}
}
// flip replaces the active buffer with the background buffer
// this is a delayed flip that simply increases the buffer counter;
// the actual flip will be performed the next time we call `buffer.fill`
func (b *buffer) flip() {
b.flipcnt += 1
}
// fill reads into the buffer until at least _need_ bytes are in it
func (b *buffer) fill(need int) error {
n := b.length
// fill data into its double-buffering target: if we've called
// flip on this buffer, we'll be copying to the background buffer,
// and then filling it with network data; otherwise we'll just move
// the contents of the current buffer to the front before filling it
dest := b.dbuf[b.flipcnt&1]
// grow buffer if necessary to fit the whole packet.
if need > len(dest) {
// Round up to the next multiple of the default size
dest = make([]byte, ((need/defaultBufSize)+1)*defaultBufSize)
// if the allocated buffer is not too large, move it to backing storage
// to prevent extra allocations on applications that perform large reads
if len(dest) <= maxCachedBufSize {
b.dbuf[b.flipcnt&1] = dest
}
}
// if we're filling the fg buffer, move the existing data to the start of it.
// if we're filling the bg buffer, copy over the data
if n > 0 {
copy(dest[:n], b.buf[b.idx:])
}
b.buf = dest
b.idx = 0
for {
if b.timeout > 0 {
if err := b.nc.SetReadDeadline(time.Now().Add(b.timeout)); err != nil {
return err
}
}
nn, err := b.nc.Read(b.buf[n:])
n += nn
switch err {
case nil:
if n < need {
continue
}
b.length = n
return nil
case io.EOF:
if n >= need {
b.length = n
return nil
}
return io.ErrUnexpectedEOF
default:
return err
}
}
}
// returns next N bytes from buffer.
// The returned slice is only guaranteed to be valid until the next read
func (b *buffer) readNext(need int) ([]byte, error) {
if b.length < need {
// refill
if err := b.fill(need); err != nil {
return nil, err
}
}
offset := b.idx
b.idx += need
b.length -= need
return b.buf[offset:b.idx], nil
}
// takeBuffer returns a buffer with the requested size.
// If possible, a slice from the existing buffer is returned.
// Otherwise a bigger buffer is made.
// Only one buffer (total) can be used at a time.
func (b *buffer) takeBuffer(length int) ([]byte, error) {
if b.length > 0 {
return nil, ErrBusyBuffer
}
// test (cheap) general case first
if length <= cap(b.buf) {
return b.buf[:length], nil
}
if length < maxPacketSize {
b.buf = make([]byte, length)
return b.buf, nil
}
// buffer is larger than we want to store.
return make([]byte, length), nil
}
// takeSmallBuffer is shortcut which can be used if length is
// known to be smaller than defaultBufSize.
// Only one buffer (total) can be used at a time.
func (b *buffer) takeSmallBuffer(length int) ([]byte, error) {
if b.length > 0 {
return nil, ErrBusyBuffer
}
return b.buf[:length], nil
}
// takeCompleteBuffer returns the complete existing buffer.
// This can be used if the necessary buffer size is unknown.
// cap and len of the returned buffer will be equal.
// Only one buffer (total) can be used at a time.
func (b *buffer) takeCompleteBuffer() ([]byte, error) {
if b.length > 0 {
return nil, ErrBusyBuffer
}
return b.buf, nil
}
// store stores buf, an updated buffer, if its suitable to do so.
func (b *buffer) store(buf []byte) error {
if b.length > 0 {
return ErrBusyBuffer
} else if cap(buf) <= maxPacketSize && cap(buf) > cap(b.buf) {
b.buf = buf[:cap(buf)]
}
return nil
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2014 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
const defaultCollation = "utf8mb4_general_ci"
const binaryCollation = "binary"
// A list of available collations mapped to the internal ID.
// To update this map use the following MySQL query:
// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS WHERE ID<256 ORDER BY ID
//
// Handshake packet have only 1 byte for collation_id. So we can't use collations with ID > 255.
//
// ucs2, utf16, and utf32 can't be used for connection charset.
// https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset
// They are commented out to reduce this map.
var collations = map[string]byte{
"big5_chinese_ci": 1,
"latin2_czech_cs": 2,
"dec8_swedish_ci": 3,
"cp850_general_ci": 4,
"latin1_german1_ci": 5,
"hp8_english_ci": 6,
"koi8r_general_ci": 7,
"latin1_swedish_ci": 8,
"latin2_general_ci": 9,
"swe7_swedish_ci": 10,
"ascii_general_ci": 11,
"ujis_japanese_ci": 12,
"sjis_japanese_ci": 13,
"cp1251_bulgarian_ci": 14,
"latin1_danish_ci": 15,
"hebrew_general_ci": 16,
"tis620_thai_ci": 18,
"euckr_korean_ci": 19,
"latin7_estonian_cs": 20,
"latin2_hungarian_ci": 21,
"koi8u_general_ci": 22,
"cp1251_ukrainian_ci": 23,
"gb2312_chinese_ci": 24,
"greek_general_ci": 25,
"cp1250_general_ci": 26,
"latin2_croatian_ci": 27,
"gbk_chinese_ci": 28,
"cp1257_lithuanian_ci": 29,
"latin5_turkish_ci": 30,
"latin1_german2_ci": 31,
"armscii8_general_ci": 32,
"utf8_general_ci": 33,
"cp1250_czech_cs": 34,
//"ucs2_general_ci": 35,
"cp866_general_ci": 36,
"keybcs2_general_ci": 37,
"macce_general_ci": 38,
"macroman_general_ci": 39,
"cp852_general_ci": 40,
"latin7_general_ci": 41,
"latin7_general_cs": 42,
"macce_bin": 43,
"cp1250_croatian_ci": 44,
"utf8mb4_general_ci": 45,
"utf8mb4_bin": 46,
"latin1_bin": 47,
"latin1_general_ci": 48,
"latin1_general_cs": 49,
"cp1251_bin": 50,
"cp1251_general_ci": 51,
"cp1251_general_cs": 52,
"macroman_bin": 53,
//"utf16_general_ci": 54,
//"utf16_bin": 55,
//"utf16le_general_ci": 56,
"cp1256_general_ci": 57,
"cp1257_bin": 58,
"cp1257_general_ci": 59,
//"utf32_general_ci": 60,
//"utf32_bin": 61,
//"utf16le_bin": 62,
"binary": 63,
"armscii8_bin": 64,
"ascii_bin": 65,
"cp1250_bin": 66,
"cp1256_bin": 67,
"cp866_bin": 68,
"dec8_bin": 69,
"greek_bin": 70,
"hebrew_bin": 71,
"hp8_bin": 72,
"keybcs2_bin": 73,
"koi8r_bin": 74,
"koi8u_bin": 75,
"utf8_tolower_ci": 76,
"latin2_bin": 77,
"latin5_bin": 78,
"latin7_bin": 79,
"cp850_bin": 80,
"cp852_bin": 81,
"swe7_bin": 82,
"utf8_bin": 83,
"big5_bin": 84,
"euckr_bin": 85,
"gb2312_bin": 86,
"gbk_bin": 87,
"sjis_bin": 88,
"tis620_bin": 89,
//"ucs2_bin": 90,
"ujis_bin": 91,
"geostd8_general_ci": 92,
"geostd8_bin": 93,
"latin1_spanish_ci": 94,
"cp932_japanese_ci": 95,
"cp932_bin": 96,
"eucjpms_japanese_ci": 97,
"eucjpms_bin": 98,
"cp1250_polish_ci": 99,
//"utf16_unicode_ci": 101,
//"utf16_icelandic_ci": 102,
//"utf16_latvian_ci": 103,
//"utf16_romanian_ci": 104,
//"utf16_slovenian_ci": 105,
//"utf16_polish_ci": 106,
//"utf16_estonian_ci": 107,
//"utf16_spanish_ci": 108,
//"utf16_swedish_ci": 109,
//"utf16_turkish_ci": 110,
//"utf16_czech_ci": 111,
//"utf16_danish_ci": 112,
//"utf16_lithuanian_ci": 113,
//"utf16_slovak_ci": 114,
//"utf16_spanish2_ci": 115,
//"utf16_roman_ci": 116,
//"utf16_persian_ci": 117,
//"utf16_esperanto_ci": 118,
//"utf16_hungarian_ci": 119,
//"utf16_sinhala_ci": 120,
//"utf16_german2_ci": 121,
//"utf16_croatian_ci": 122,
//"utf16_unicode_520_ci": 123,
//"utf16_vietnamese_ci": 124,
//"ucs2_unicode_ci": 128,
//"ucs2_icelandic_ci": 129,
//"ucs2_latvian_ci": 130,
//"ucs2_romanian_ci": 131,
//"ucs2_slovenian_ci": 132,
//"ucs2_polish_ci": 133,
//"ucs2_estonian_ci": 134,
//"ucs2_spanish_ci": 135,
//"ucs2_swedish_ci": 136,
//"ucs2_turkish_ci": 137,
//"ucs2_czech_ci": 138,
//"ucs2_danish_ci": 139,
//"ucs2_lithuanian_ci": 140,
//"ucs2_slovak_ci": 141,
//"ucs2_spanish2_ci": 142,
//"ucs2_roman_ci": 143,
//"ucs2_persian_ci": 144,
//"ucs2_esperanto_ci": 145,
//"ucs2_hungarian_ci": 146,
//"ucs2_sinhala_ci": 147,
//"ucs2_german2_ci": 148,
//"ucs2_croatian_ci": 149,
//"ucs2_unicode_520_ci": 150,
//"ucs2_vietnamese_ci": 151,
//"ucs2_general_mysql500_ci": 159,
//"utf32_unicode_ci": 160,
//"utf32_icelandic_ci": 161,
//"utf32_latvian_ci": 162,
//"utf32_romanian_ci": 163,
//"utf32_slovenian_ci": 164,
//"utf32_polish_ci": 165,
//"utf32_estonian_ci": 166,
//"utf32_spanish_ci": 167,
//"utf32_swedish_ci": 168,
//"utf32_turkish_ci": 169,
//"utf32_czech_ci": 170,
//"utf32_danish_ci": 171,
//"utf32_lithuanian_ci": 172,
//"utf32_slovak_ci": 173,
//"utf32_spanish2_ci": 174,
//"utf32_roman_ci": 175,
//"utf32_persian_ci": 176,
//"utf32_esperanto_ci": 177,
//"utf32_hungarian_ci": 178,
//"utf32_sinhala_ci": 179,
//"utf32_german2_ci": 180,
//"utf32_croatian_ci": 181,
//"utf32_unicode_520_ci": 182,
//"utf32_vietnamese_ci": 183,
"utf8_unicode_ci": 192,
"utf8_icelandic_ci": 193,
"utf8_latvian_ci": 194,
"utf8_romanian_ci": 195,
"utf8_slovenian_ci": 196,
"utf8_polish_ci": 197,
"utf8_estonian_ci": 198,
"utf8_spanish_ci": 199,
"utf8_swedish_ci": 200,
"utf8_turkish_ci": 201,
"utf8_czech_ci": 202,
"utf8_danish_ci": 203,
"utf8_lithuanian_ci": 204,
"utf8_slovak_ci": 205,
"utf8_spanish2_ci": 206,
"utf8_roman_ci": 207,
"utf8_persian_ci": 208,
"utf8_esperanto_ci": 209,
"utf8_hungarian_ci": 210,
"utf8_sinhala_ci": 211,
"utf8_german2_ci": 212,
"utf8_croatian_ci": 213,
"utf8_unicode_520_ci": 214,
"utf8_vietnamese_ci": 215,
"utf8_general_mysql500_ci": 223,
"utf8mb4_unicode_ci": 224,
"utf8mb4_icelandic_ci": 225,
"utf8mb4_latvian_ci": 226,
"utf8mb4_romanian_ci": 227,
"utf8mb4_slovenian_ci": 228,
"utf8mb4_polish_ci": 229,
"utf8mb4_estonian_ci": 230,
"utf8mb4_spanish_ci": 231,
"utf8mb4_swedish_ci": 232,
"utf8mb4_turkish_ci": 233,
"utf8mb4_czech_ci": 234,
"utf8mb4_danish_ci": 235,
"utf8mb4_lithuanian_ci": 236,
"utf8mb4_slovak_ci": 237,
"utf8mb4_spanish2_ci": 238,
"utf8mb4_roman_ci": 239,
"utf8mb4_persian_ci": 240,
"utf8mb4_esperanto_ci": 241,
"utf8mb4_hungarian_ci": 242,
"utf8mb4_sinhala_ci": 243,
"utf8mb4_german2_ci": 244,
"utf8mb4_croatian_ci": 245,
"utf8mb4_unicode_520_ci": 246,
"utf8mb4_vietnamese_ci": 247,
"gb18030_chinese_ci": 248,
"gb18030_bin": 249,
"gb18030_unicode_520_ci": 250,
"utf8mb4_0900_ai_ci": 255,
}
// A denylist of collations which is unsafe to interpolate parameters.
// These multibyte encodings may contains 0x5c (`\`) in their trailing bytes.
var unsafeCollations = map[string]bool{
"big5_chinese_ci": true,
"sjis_japanese_ci": true,
"gbk_chinese_ci": true,
"big5_bin": true,
"gb2312_bin": true,
"gbk_bin": true,
"sjis_bin": true,
"cp932_japanese_ci": true,
"cp932_bin": true,
"gb18030_chinese_ci": true,
"gb18030_bin": true,
"gb18030_unicode_520_ci": true,
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos
package mysql
import (
"errors"
"io"
"net"
"syscall"
)
var errUnexpectedRead = errors.New("unexpected read from socket")
func connCheck(conn net.Conn) error {
var sysErr error
sysConn, ok := conn.(syscall.Conn)
if !ok {
return nil
}
rawConn, err := sysConn.SyscallConn()
if err != nil {
return err
}
err = rawConn.Read(func(fd uintptr) bool {
var buf [1]byte
n, err := syscall.Read(int(fd), buf[:])
switch {
case n == 0 && err == nil:
sysErr = io.EOF
case n > 0:
sysErr = errUnexpectedRead
case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK:
sysErr = nil
default:
sysErr = err
}
return true
})
if err != nil {
return err
}
return sysErr
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build !linux,!darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!illumos
package mysql
import "net"
func connCheck(conn net.Conn) error {
return nil
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos
package mysql
import (
"testing"
"time"
)
func TestStaleConnectionChecks(t *testing.T) {
runTests(t, dsn, func(dbt *DBTest) {
dbt.mustExec("SET @@SESSION.wait_timeout = 2")
if err := dbt.db.Ping(); err != nil {
dbt.Fatal(err)
}
// wait for MySQL to close our connection
time.Sleep(3 * time.Second)
tx, err := dbt.db.Begin()
if err != nil {
dbt.Fatal(err)
}
if err := tx.Rollback(); err != nil {
dbt.Fatal(err)
}
})
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"context"
"database/sql/driver"
"encoding/json"
"errors"
"net"
"testing"
)
func TestInterpolateParams(t *testing.T) {
mc := &mysqlConn{
buf: newBuffer(nil),
maxAllowedPacket: maxPacketSize,
cfg: &Config{
InterpolateParams: true,
},
}
q, err := mc.interpolateParams("SELECT ?+?", []driver.Value{int64(42), "gopher"})
if err != nil {
t.Errorf("Expected err=nil, got %#v", err)
return
}
expected := `SELECT 42+'gopher'`
if q != expected {
t.Errorf("Expected: %q\nGot: %q", expected, q)
}
}
func TestInterpolateParamsJSONRawMessage(t *testing.T) {
mc := &mysqlConn{
buf: newBuffer(nil),
maxAllowedPacket: maxPacketSize,
cfg: &Config{
InterpolateParams: true,
},
}
buf, err := json.Marshal(struct {
Value int `json:"value"`
}{Value: 42})
if err != nil {
t.Errorf("Expected err=nil, got %#v", err)
return
}
q, err := mc.interpolateParams("SELECT ?", []driver.Value{json.RawMessage(buf)})
if err != nil {
t.Errorf("Expected err=nil, got %#v", err)
return
}
expected := `SELECT '{\"value\":42}'`
if q != expected {
t.Errorf("Expected: %q\nGot: %q", expected, q)
}
}
func TestInterpolateParamsTooManyPlaceholders(t *testing.T) {
mc := &mysqlConn{
buf: newBuffer(nil),
maxAllowedPacket: maxPacketSize,
cfg: &Config{
InterpolateParams: true,
},
}
q, err := mc.interpolateParams("SELECT ?+?", []driver.Value{int64(42)})
if err != driver.ErrSkip {
t.Errorf("Expected err=driver.ErrSkip, got err=%#v, q=%#v", err, q)
}
}
// We don't support placeholder in string literal for now.
// https://github.com/go-sql-driver/mysql/pull/490
func TestInterpolateParamsPlaceholderInString(t *testing.T) {
mc := &mysqlConn{
buf: newBuffer(nil),
maxAllowedPacket: maxPacketSize,
cfg: &Config{
InterpolateParams: true,
},
}
q, err := mc.interpolateParams("SELECT 'abc?xyz',?", []driver.Value{int64(42)})
// When InterpolateParams support string literal, this should return `"SELECT 'abc?xyz', 42`
if err != driver.ErrSkip {
t.Errorf("Expected err=driver.ErrSkip, got err=%#v, q=%#v", err, q)
}
}
func TestInterpolateParamsUint64(t *testing.T) {
mc := &mysqlConn{
buf: newBuffer(nil),
maxAllowedPacket: maxPacketSize,
cfg: &Config{
InterpolateParams: true,
},
}
q, err := mc.interpolateParams("SELECT ?", []driver.Value{uint64(42)})
if err != nil {
t.Errorf("Expected err=nil, got err=%#v, q=%#v", err, q)
}
if q != "SELECT 42" {
t.Errorf("Expected uint64 interpolation to work, got q=%#v", q)
}
}
func TestCheckNamedValue(t *testing.T) {
value := driver.NamedValue{Value: ^uint64(0)}
x := &mysqlConn{}
err := x.CheckNamedValue(&value)
if err != nil {
t.Fatal("uint64 high-bit not convertible", err)
}
if value.Value != ^uint64(0) {
t.Fatalf("uint64 high-bit converted, got %#v %T", value.Value, value.Value)
}
}
// TestCleanCancel tests passed context is cancelled at start.
// No packet should be sent. Connection should keep current status.
func TestCleanCancel(t *testing.T) {
mc := &mysqlConn{
closech: make(chan struct{}),
}
mc.startWatcher()
defer mc.cleanup()
ctx, cancel := context.WithCancel(context.Background())
cancel()
for i := 0; i < 3; i++ { // Repeat same behavior
err := mc.Ping(ctx)
if err != context.Canceled {
t.Errorf("expected context.Canceled, got %#v", err)
}
if mc.closed.IsSet() {
t.Error("expected mc is not closed, closed actually")
}
if mc.watching {
t.Error("expected watching is false, but true")
}
}
}
func TestPingMarkBadConnection(t *testing.T) {
nc := badConnection{err: errors.New("boom")}
ms := &mysqlConn{
netConn: nc,
buf: newBuffer(nc),
maxAllowedPacket: defaultMaxAllowedPacket,
}
err := ms.Ping(context.Background())
if err != driver.ErrBadConn {
t.Errorf("expected driver.ErrBadConn, got %#v", err)
}
}
func TestPingErrInvalidConn(t *testing.T) {
nc := badConnection{err: errors.New("failed to write"), n: 10}
ms := &mysqlConn{
netConn: nc,
buf: newBuffer(nc),
maxAllowedPacket: defaultMaxAllowedPacket,
closech: make(chan struct{}),
}
err := ms.Ping(context.Background())
if err != ErrInvalidConn {
t.Errorf("expected ErrInvalidConn, got %#v", err)
}
}
type badConnection struct {
n int
err error
net.Conn
}
func (bc badConnection) Write(b []byte) (n int, err error) {
return bc.n, bc.err
}
func (bc badConnection) Close() error {
return nil
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"context"
"database/sql/driver"
"net"
)
type connector struct {
cfg *Config // immutable private copy.
}
// Connect implements driver.Connector interface.
// Connect returns a connection to the database.
func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
var err error
// New mysqlConn
mc := &mysqlConn{
maxAllowedPacket: maxPacketSize,
maxWriteSize: maxPacketSize - 1,
closech: make(chan struct{}),
cfg: c.cfg,
}
mc.parseTime = mc.cfg.ParseTime
// Connect to Server
dialsLock.RLock()
dial, ok := dials[mc.cfg.Net]
dialsLock.RUnlock()
if ok {
dctx := ctx
if mc.cfg.Timeout > 0 {
var cancel context.CancelFunc
dctx, cancel = context.WithTimeout(ctx, c.cfg.Timeout)
defer cancel()
}
mc.netConn, err = dial(dctx, mc.cfg.Addr)
} else {
nd := net.Dialer{Timeout: mc.cfg.Timeout}
mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr)
}
if err != nil {
return nil, err
}
// Enable TCP Keepalives on TCP connections
if tc, ok := mc.netConn.(*net.TCPConn); ok {
if err := tc.SetKeepAlive(true); err != nil {
// Don't send COM_QUIT before handshake.
mc.netConn.Close()
mc.netConn = nil
return nil, err
}
}
// Call startWatcher for context support (From Go 1.8)
mc.startWatcher()
if err := mc.watchCancel(ctx); err != nil {
mc.cleanup()
return nil, err
}
defer mc.finish()
mc.buf = newBuffer(mc.netConn)
// Set I/O timeouts
mc.buf.timeout = mc.cfg.ReadTimeout
mc.writeTimeout = mc.cfg.WriteTimeout
// Reading Handshake Initialization Packet
authData, plugin, err := mc.readHandshakePacket()
if err != nil {
mc.cleanup()
return nil, err
}
if plugin == "" {
plugin = defaultAuthPlugin
}
// Send Client Authentication Packet
authResp, err := mc.auth(authData, plugin)
if err != nil {
// try the default auth plugin, if using the requested plugin failed
errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error())
plugin = defaultAuthPlugin
authResp, err = mc.auth(authData, plugin)
if err != nil {
mc.cleanup()
return nil, err
}
}
if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil {
mc.cleanup()
return nil, err
}
// Handle response to auth packet, switch methods if possible
if err = mc.handleAuthResult(authData, plugin); err != nil {
// Authentication failed and MySQL has already closed the connection
// (https://dev.mysql.com/doc/internals/en/authentication-fails.html).
// Do not send COM_QUIT, just cleanup and return the error.
mc.cleanup()
return nil, err
}
if mc.cfg.MaxAllowedPacket > 0 {
mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket
} else {
// Get max allowed packet size
maxap, err := mc.getSystemVar("max_allowed_packet")
if err != nil {
mc.Close()
return nil, err
}
mc.maxAllowedPacket = stringToInt(maxap) - 1
}
if mc.maxAllowedPacket < maxPacketSize {
mc.maxWriteSize = mc.maxAllowedPacket
}
// Handle DSN Params
err = mc.handleParams()
if err != nil {
mc.Close()
return nil, err
}
return mc, nil
}
// Driver implements driver.Connector interface.
// Driver returns &MySQLDriver{}.
func (c *connector) Driver() driver.Driver {
return &MySQLDriver{}
}
package mysql
import (
"context"
"net"
"testing"
"time"
)
func TestConnectorReturnsTimeout(t *testing.T) {
connector := &connector{&Config{
Net: "tcp",
Addr: "1.1.1.1:1234",
Timeout: 10 * time.Millisecond,
}}
_, err := connector.Connect(context.Background())
if err == nil {
t.Fatal("error expected")
}
if nerr, ok := err.(*net.OpError); ok {
expected := "dial tcp 1.1.1.1:1234: i/o timeout"
if nerr.Error() != expected {
t.Fatalf("expected %q, got %q", expected, nerr.Error())
}
} else {
t.Fatalf("expected %T, got %T", nerr, err)
}
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
const (
defaultAuthPlugin = "mysql_native_password"
defaultMaxAllowedPacket = 4 << 20 // 4 MiB
minProtocolVersion = 10
maxPacketSize = 1<<24 - 1
timeFormat = "2006-01-02 15:04:05.999999"
)
// MySQL constants documentation:
// http://dev.mysql.com/doc/internals/en/client-server-protocol.html
const (
iOK byte = 0x00
iAuthMoreData byte = 0x01
iLocalInFile byte = 0xfb
iEOF byte = 0xfe
iERR byte = 0xff
)
// https://dev.mysql.com/doc/internals/en/capability-flags.html#packet-Protocol::CapabilityFlags
type clientFlag uint32
const (
clientLongPassword clientFlag = 1 << iota
clientFoundRows
clientLongFlag
clientConnectWithDB
clientNoSchema
clientCompress
clientODBC
clientLocalFiles
clientIgnoreSpace
clientProtocol41
clientInteractive
clientSSL
clientIgnoreSIGPIPE
clientTransactions
clientReserved
clientSecureConn
clientMultiStatements
clientMultiResults
clientPSMultiResults
clientPluginAuth
clientConnectAttrs
clientPluginAuthLenEncClientData
clientCanHandleExpiredPasswords
clientSessionTrack
clientDeprecateEOF
)
const (
comQuit byte = iota + 1
comInitDB
comQuery
comFieldList
comCreateDB
comDropDB
comRefresh
comShutdown
comStatistics
comProcessInfo
comConnect
comProcessKill
comDebug
comPing
comTime
comDelayedInsert
comChangeUser
comBinlogDump
comTableDump
comConnectOut
comRegisterSlave
comStmtPrepare
comStmtExecute
comStmtSendLongData
comStmtClose
comStmtReset
comSetOption
comStmtFetch
)
// https://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnType
type fieldType byte
const (
fieldTypeDecimal fieldType = iota
fieldTypeTiny
fieldTypeShort
fieldTypeLong
fieldTypeFloat
fieldTypeDouble
fieldTypeNULL
fieldTypeTimestamp
fieldTypeLongLong
fieldTypeInt24
fieldTypeDate
fieldTypeTime
fieldTypeDateTime
fieldTypeYear
fieldTypeNewDate
fieldTypeVarChar
fieldTypeBit
)
const (
fieldTypeJSON fieldType = iota + 0xf5
fieldTypeNewDecimal
fieldTypeEnum
fieldTypeSet
fieldTypeTinyBLOB
fieldTypeMediumBLOB
fieldTypeLongBLOB
fieldTypeBLOB
fieldTypeVarString
fieldTypeString
fieldTypeGeometry
)
type fieldFlag uint16
const (
flagNotNULL fieldFlag = 1 << iota
flagPriKey
flagUniqueKey
flagMultipleKey
flagBLOB
flagUnsigned
flagZeroFill
flagBinary
flagEnum
flagAutoIncrement
flagTimestamp
flagSet
flagUnknown1
flagUnknown2
flagUnknown3
flagUnknown4
)
// http://dev.mysql.com/doc/internals/en/status-flags.html
type statusFlag uint16
const (
statusInTrans statusFlag = 1 << iota
statusInAutocommit
statusReserved // Not in documentation
statusMoreResultsExists
statusNoGoodIndexUsed
statusNoIndexUsed
statusCursorExists
statusLastRowSent
statusDbDropped
statusNoBackslashEscapes
statusMetadataChanged
statusQueryWasSlow
statusPsOutParams
statusInTransReadonly
statusSessionStateChanged
)
const (
cachingSha2PasswordRequestPublicKey = 2
cachingSha2PasswordFastAuthSuccess = 3
cachingSha2PasswordPerformFullAuthentication = 4
)
// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// Package mysql provides a MySQL driver for Go's database/sql package.
//
// The driver should be used via the database/sql package:
//
// import "database/sql"
// import _ "github.com/go-sql-driver/mysql"
//
// db, err := sql.Open("mysql", "user:password@/dbname")
//
// See https://github.com/go-sql-driver/mysql#usage for details
package mysql
import (
"context"
"database/sql"
"database/sql/driver"
"net"
"sync"
)
// MySQLDriver is exported to make the driver directly accessible.
// In general the driver is used via the database/sql package.
type MySQLDriver struct{}
// DialFunc is a function which can be used to establish the network connection.
// Custom dial functions must be registered with RegisterDial
//
// Deprecated: users should register a DialContextFunc instead
type DialFunc func(addr string) (net.Conn, error)
// DialContextFunc is a function which can be used to establish the network connection.
// Custom dial functions must be registered with RegisterDialContext
type DialContextFunc func(ctx context.Context, addr string) (net.Conn, error)
var (
dialsLock sync.RWMutex
dials map[string]DialContextFunc
)
// RegisterDialContext registers a custom dial function. It can then be used by the
// network address mynet(addr), where mynet is the registered new network.
// The current context for the connection and its address is passed to the dial function.
func RegisterDialContext(net string, dial DialContextFunc) {
dialsLock.Lock()
defer dialsLock.Unlock()
if dials == nil {
dials = make(map[string]DialContextFunc)
}
dials[net] = dial
}
// RegisterDial registers a custom dial function. It can then be used by the
// network address mynet(addr), where mynet is the registered new network.
// addr is passed as a parameter to the dial function.
//
// Deprecated: users should call RegisterDialContext instead
func RegisterDial(network string, dial DialFunc) {
RegisterDialContext(network, func(_ context.Context, addr string) (net.Conn, error) {
return dial(addr)
})
}
// Open new Connection.
// See https://github.com/go-sql-driver/mysql#dsn-data-source-name for how
// the DSN string is formatted
func (d MySQLDriver) Open(dsn string) (driver.Conn, error) {
cfg, err := ParseDSN(dsn)
if err != nil {
return nil, err
}
c := &connector{
cfg: cfg,
}
return c.Connect(context.Background())
}
func init() {
sql.Register("mysql", &MySQLDriver{})
}
// NewConnector returns new driver.Connector.
func NewConnector(cfg *Config) (driver.Connector, error) {
cfg = cfg.Clone()
// normalize the contents of cfg so calls to NewConnector have the same
// behavior as MySQLDriver.OpenConnector
if err := cfg.normalize(); err != nil {
return nil, err
}
return &connector{cfg: cfg}, nil
}
// OpenConnector implements driver.DriverContext.
func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) {
cfg, err := ParseDSN(dsn)
if err != nil {
return nil, err
}
return &connector{
cfg: cfg,
}, nil
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"errors"
"fmt"
"log"
"os"
)
// Various errors the driver might return. Can change between driver versions.
var (
ErrInvalidConn = errors.New("invalid connection")
ErrMalformPkt = errors.New("malformed packet")
ErrNoTLS = errors.New("TLS requested but server does not support TLS")
ErrCleartextPassword = errors.New("this user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN")
ErrNativePassword = errors.New("this user requires mysql native password authentication.")
ErrOldPassword = errors.New("this user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords")
ErrUnknownPlugin = errors.New("this authentication plugin is not supported")
ErrOldProtocol = errors.New("MySQL server does not support required protocol 41+")
ErrPktSync = errors.New("commands out of sync. You can't run this command now")
ErrPktSyncMul = errors.New("commands out of sync. Did you run multiple statements at once?")
ErrPktTooLarge = errors.New("packet for query is too large. Try adjusting the 'max_allowed_packet' variable on the server")
ErrBusyBuffer = errors.New("busy buffer")
// errBadConnNoWrite is used for connection errors where nothing was sent to the database yet.
// If this happens first in a function starting a database interaction, it should be replaced by driver.ErrBadConn
// to trigger a resend.
// See https://github.com/go-sql-driver/mysql/pull/302
errBadConnNoWrite = errors.New("bad connection")
)
var errLog = Logger(log.New(os.Stderr, "[mysql] ", log.Ldate|log.Ltime|log.Lshortfile))
// Logger is used to log critical error messages.
type Logger interface {
Print(v ...interface{})
}
// SetLogger is used to set the logger for critical errors.
// The initial logger is os.Stderr.
func SetLogger(logger Logger) error {
if logger == nil {
return errors.New("logger is nil")
}
errLog = logger
return nil
}
// MySQLError is an error type which represents a single MySQL error
type MySQLError struct {
Number uint16
Message string
}
func (me *MySQLError) Error() string {
return fmt.Sprintf("Error %d: %s", me.Number, me.Message)
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"bytes"
"log"
"testing"
)
func TestErrorsSetLogger(t *testing.T) {
previous := errLog
defer func() {
errLog = previous
}()
// set up logger
const expected = "prefix: test\n"
buffer := bytes.NewBuffer(make([]byte, 0, 64))
logger := log.New(buffer, "prefix: ", 0)
// print
SetLogger(logger)
errLog.Print("test")
// check result
if actual := buffer.String(); actual != expected {
t.Errorf("expected %q, got %q", expected, actual)
}
}
func TestErrorsStrictIgnoreNotes(t *testing.T) {
runTests(t, dsn+"&sql_notes=false", func(dbt *DBTest) {
dbt.mustExec("DROP TABLE IF EXISTS does_not_exist")
})
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2017 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"database/sql"
"reflect"
)
func (mf *mysqlField) typeDatabaseName() string {
switch mf.fieldType {
case fieldTypeBit:
return "BIT"
case fieldTypeBLOB:
if mf.charSet != collations[binaryCollation] {
return "TEXT"
}
return "BLOB"
case fieldTypeDate:
return "DATE"
case fieldTypeDateTime:
return "DATETIME"
case fieldTypeDecimal:
return "DECIMAL"
case fieldTypeDouble:
return "DOUBLE"
case fieldTypeEnum:
return "ENUM"
case fieldTypeFloat:
return "FLOAT"
case fieldTypeGeometry:
return "GEOMETRY"
case fieldTypeInt24:
return "MEDIUMINT"
case fieldTypeJSON:
return "JSON"
case fieldTypeLong:
return "INT"
case fieldTypeLongBLOB:
if mf.charSet != collations[binaryCollation] {
return "LONGTEXT"
}
return "LONGBLOB"
case fieldTypeLongLong:
return "BIGINT"
case fieldTypeMediumBLOB:
if mf.charSet != collations[binaryCollation] {
return "MEDIUMTEXT"
}
return "MEDIUMBLOB"
case fieldTypeNewDate:
return "DATE"
case fieldTypeNewDecimal:
return "DECIMAL"
case fieldTypeNULL:
return "NULL"
case fieldTypeSet:
return "SET"
case fieldTypeShort:
return "SMALLINT"
case fieldTypeString:
if mf.charSet == collations[binaryCollation] {
return "BINARY"
}
return "CHAR"
case fieldTypeTime:
return "TIME"
case fieldTypeTimestamp:
return "TIMESTAMP"
case fieldTypeTiny:
return "TINYINT"
case fieldTypeTinyBLOB:
if mf.charSet != collations[binaryCollation] {
return "TINYTEXT"
}
return "TINYBLOB"
case fieldTypeVarChar:
if mf.charSet == collations[binaryCollation] {
return "VARBINARY"
}
return "VARCHAR"
case fieldTypeVarString:
if mf.charSet == collations[binaryCollation] {
return "VARBINARY"
}
return "VARCHAR"
case fieldTypeYear:
return "YEAR"
default:
return ""
}
}
var (
scanTypeFloat32 = reflect.TypeOf(float32(0))
scanTypeFloat64 = reflect.TypeOf(float64(0))
scanTypeInt8 = reflect.TypeOf(int8(0))
scanTypeInt16 = reflect.TypeOf(int16(0))
scanTypeInt32 = reflect.TypeOf(int32(0))
scanTypeInt64 = reflect.TypeOf(int64(0))
scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{})
scanTypeNullInt = reflect.TypeOf(sql.NullInt64{})
scanTypeNullTime = reflect.TypeOf(nullTime{})
scanTypeUint8 = reflect.TypeOf(uint8(0))
scanTypeUint16 = reflect.TypeOf(uint16(0))
scanTypeUint32 = reflect.TypeOf(uint32(0))
scanTypeUint64 = reflect.TypeOf(uint64(0))
scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{})
scanTypeUnknown = reflect.TypeOf(new(interface{}))
)
type mysqlField struct {
tableName string
name string
length uint32
flags fieldFlag
fieldType fieldType
decimals byte
charSet uint8
}
func (mf *mysqlField) scanType() reflect.Type {
switch mf.fieldType {
case fieldTypeTiny:
if mf.flags&flagNotNULL != 0 {
if mf.flags&flagUnsigned != 0 {
return scanTypeUint8
}
return scanTypeInt8
}
return scanTypeNullInt
case fieldTypeShort, fieldTypeYear:
if mf.flags&flagNotNULL != 0 {
if mf.flags&flagUnsigned != 0 {
return scanTypeUint16
}
return scanTypeInt16
}
return scanTypeNullInt
case fieldTypeInt24, fieldTypeLong:
if mf.flags&flagNotNULL != 0 {
if mf.flags&flagUnsigned != 0 {
return scanTypeUint32
}
return scanTypeInt32
}
return scanTypeNullInt
case fieldTypeLongLong:
if mf.flags&flagNotNULL != 0 {
if mf.flags&flagUnsigned != 0 {
return scanTypeUint64
}
return scanTypeInt64
}
return scanTypeNullInt
case fieldTypeFloat:
if mf.flags&flagNotNULL != 0 {
return scanTypeFloat32
}
return scanTypeNullFloat
case fieldTypeDouble:
if mf.flags&flagNotNULL != 0 {
return scanTypeFloat64
}
return scanTypeNullFloat
case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar,
fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB,
fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB,
fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON,
fieldTypeTime:
return scanTypeRawBytes
case fieldTypeDate, fieldTypeNewDate,
fieldTypeTimestamp, fieldTypeDateTime:
// NullTime is always returned for more consistent behavior as it can
// handle both cases of parseTime regardless if the field is nullable.
return scanTypeNullTime
default:
return scanTypeUnknown
}
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package.
//
// Copyright 2020 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build gofuzz
package mysql
import (
"database/sql"
)
func Fuzz(data []byte) int {
db, err := sql.Open("mysql", string(data))
if err != nil {
return 0
}
db.Close()
return 1
}
module github.com/go-sql-driver/mysql
go 1.10
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"fmt"
"io"
"os"
"strings"
"sync"
)
var (
fileRegister map[string]bool
fileRegisterLock sync.RWMutex
readerRegister map[string]func() io.Reader
readerRegisterLock sync.RWMutex
)
// RegisterLocalFile adds the given file to the file allowlist,
// so that it can be used by "LOAD DATA LOCAL INFILE <filepath>".
// Alternatively you can allow the use of all local files with
// the DSN parameter 'allowAllFiles=true'
//
// filePath := "/home/gopher/data.csv"
// mysql.RegisterLocalFile(filePath)
// err := db.Exec("LOAD DATA LOCAL INFILE '" + filePath + "' INTO TABLE foo")
// if err != nil {
// ...
//
func RegisterLocalFile(filePath string) {
fileRegisterLock.Lock()
// lazy map init
if fileRegister == nil {
fileRegister = make(map[string]bool)
}
fileRegister[strings.Trim(filePath, `"`)] = true
fileRegisterLock.Unlock()
}
// DeregisterLocalFile removes the given filepath from the allowlist.
func DeregisterLocalFile(filePath string) {
fileRegisterLock.Lock()
delete(fileRegister, strings.Trim(filePath, `"`))
fileRegisterLock.Unlock()
}
// RegisterReaderHandler registers a handler function which is used
// to receive a io.Reader.
// The Reader can be used by "LOAD DATA LOCAL INFILE Reader::<name>".
// If the handler returns a io.ReadCloser Close() is called when the
// request is finished.
//
// mysql.RegisterReaderHandler("data", func() io.Reader {
// var csvReader io.Reader // Some Reader that returns CSV data
// ... // Open Reader here
// return csvReader
// })
// err := db.Exec("LOAD DATA LOCAL INFILE 'Reader::data' INTO TABLE foo")
// if err != nil {
// ...
//
func RegisterReaderHandler(name string, handler func() io.Reader) {
readerRegisterLock.Lock()
// lazy map init
if readerRegister == nil {
readerRegister = make(map[string]func() io.Reader)
}
readerRegister[name] = handler
readerRegisterLock.Unlock()
}
// DeregisterReaderHandler removes the ReaderHandler function with
// the given name from the registry.
func DeregisterReaderHandler(name string) {
readerRegisterLock.Lock()
delete(readerRegister, name)
readerRegisterLock.Unlock()
}
func deferredClose(err *error, closer io.Closer) {
closeErr := closer.Close()
if *err == nil {
*err = closeErr
}
}
func (mc *mysqlConn) handleInFileRequest(name string) (err error) {
var rdr io.Reader
var data []byte
packetSize := 16 * 1024 // 16KB is small enough for disk readahead and large enough for TCP
if mc.maxWriteSize < packetSize {
packetSize = mc.maxWriteSize
}
if idx := strings.Index(name, "Reader::"); idx == 0 || (idx > 0 && name[idx-1] == '/') { // io.Reader
// The server might return an an absolute path. See issue #355.
name = name[idx+8:]
readerRegisterLock.RLock()
handler, inMap := readerRegister[name]
readerRegisterLock.RUnlock()
if inMap {
rdr = handler()
if rdr != nil {
if cl, ok := rdr.(io.Closer); ok {
defer deferredClose(&err, cl)
}
} else {
err = fmt.Errorf("Reader '%s' is <nil>", name)
}
} else {
err = fmt.Errorf("Reader '%s' is not registered", name)
}
} else { // File
name = strings.Trim(name, `"`)
fileRegisterLock.RLock()
fr := fileRegister[name]
fileRegisterLock.RUnlock()
if mc.cfg.AllowAllFiles || fr {
var file *os.File
var fi os.FileInfo
if file, err = os.Open(name); err == nil {
defer deferredClose(&err, file)
// get file size
if fi, err = file.Stat(); err == nil {
rdr = file
if fileSize := int(fi.Size()); fileSize < packetSize {
packetSize = fileSize
}
}
}
} else {
err = fmt.Errorf("local file '%s' is not registered", name)
}
}
// send content packets
// if packetSize == 0, the Reader contains no data
if err == nil && packetSize > 0 {
data := make([]byte, 4+packetSize)
var n int
for err == nil {
n, err = rdr.Read(data[4:])
if n > 0 {
if ioErr := mc.writePacket(data[:4+n]); ioErr != nil {
return ioErr
}
}
}
if err == io.EOF {
err = nil
}
}
// send empty packet (termination)
if data == nil {
data = make([]byte, 4)
}
if ioErr := mc.writePacket(data[:4]); ioErr != nil {
return ioErr
}
// read OK packet
if err == nil {
return mc.readResultOK()
}
mc.readPacket()
return err
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"database/sql/driver"
"fmt"
"time"
)
// Scan implements the Scanner interface.
// The value type must be time.Time or string / []byte (formatted time-string),
// otherwise Scan fails.
func (nt *NullTime) Scan(value interface{}) (err error) {
if value == nil {
nt.Time, nt.Valid = time.Time{}, false
return
}
switch v := value.(type) {
case time.Time:
nt.Time, nt.Valid = v, true
return
case []byte:
nt.Time, err = parseDateTime(v, time.UTC)
nt.Valid = (err == nil)
return
case string:
nt.Time, err = parseDateTime([]byte(v), time.UTC)
nt.Valid = (err == nil)
return
}
nt.Valid = false
return fmt.Errorf("Can't convert %T to time.Time", value)
}
// Value implements the driver Valuer interface.
func (nt NullTime) Value() (driver.Value, error) {
if !nt.Valid {
return nil, nil
}
return nt.Time, nil
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build go1.13
package mysql
import (
"database/sql"
)
// NullTime represents a time.Time that may be NULL.
// NullTime implements the Scanner interface so
// it can be used as a scan destination:
//
// var nt NullTime
// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt)
// ...
// if nt.Valid {
// // use nt.Time
// } else {
// // NULL value
// }
//
// This NullTime implementation is not driver-specific
//
// Deprecated: NullTime doesn't honor the loc DSN parameter.
// NullTime.Scan interprets a time as UTC, not the loc DSN parameter.
// Use sql.NullTime instead.
type NullTime sql.NullTime
// for internal use.
// the mysql package uses sql.NullTime if it is available.
// if not, the package uses mysql.NullTime.
type nullTime = sql.NullTime // sql.NullTime is available
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build !go1.13
package mysql
import (
"time"
)
// NullTime represents a time.Time that may be NULL.
// NullTime implements the Scanner interface so
// it can be used as a scan destination:
//
// var nt NullTime
// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt)
// ...
// if nt.Valid {
// // use nt.Time
// } else {
// // NULL value
// }
//
// This NullTime implementation is not driver-specific
type NullTime struct {
Time time.Time
Valid bool // Valid is true if Time is not NULL
}
// for internal use.
// the mysql package uses sql.NullTime if it is available.
// if not, the package uses mysql.NullTime.
type nullTime = NullTime // sql.NullTime is not available
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"database/sql"
"database/sql/driver"
"testing"
"time"
)
var (
// Check implementation of interfaces
_ driver.Valuer = NullTime{}
_ sql.Scanner = (*NullTime)(nil)
)
func TestScanNullTime(t *testing.T) {
var scanTests = []struct {
in interface{}
error bool
valid bool
time time.Time
}{
{tDate, false, true, tDate},
{sDate, false, true, tDate},
{[]byte(sDate), false, true, tDate},
{tDateTime, false, true, tDateTime},
{sDateTime, false, true, tDateTime},
{[]byte(sDateTime), false, true, tDateTime},
{tDate0, false, true, tDate0},
{sDate0, false, true, tDate0},
{[]byte(sDate0), false, true, tDate0},
{sDateTime0, false, true, tDate0},
{[]byte(sDateTime0), false, true, tDate0},
{"", true, false, tDate0},
{"1234", true, false, tDate0},
{0, true, false, tDate0},
}
var nt = NullTime{}
var err error
for _, tst := range scanTests {
err = nt.Scan(tst.in)
if (err != nil) != tst.error {
t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil))
}
if nt.Valid != tst.valid {
t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid)
}
if nt.Time != tst.time {
t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time)
}
}
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"bytes"
"errors"
"net"
"testing"
"time"
)
var (
errConnClosed = errors.New("connection is closed")
errConnTooManyReads = errors.New("too many reads")
errConnTooManyWrites = errors.New("too many writes")
)
// struct to mock a net.Conn for testing purposes
type mockConn struct {
laddr net.Addr
raddr net.Addr
data []byte
written []byte
queuedReplies [][]byte
closed bool
read int
reads int
writes int
maxReads int
maxWrites int
}
func (m *mockConn) Read(b []byte) (n int, err error) {
if m.closed {
return 0, errConnClosed
}
m.reads++
if m.maxReads > 0 && m.reads > m.maxReads {
return 0, errConnTooManyReads
}
n = copy(b, m.data)
m.read += n
m.data = m.data[n:]
return
}
func (m *mockConn) Write(b []byte) (n int, err error) {
if m.closed {
return 0, errConnClosed
}
m.writes++
if m.maxWrites > 0 && m.writes > m.maxWrites {
return 0, errConnTooManyWrites
}
n = len(b)
m.written = append(m.written, b...)
if n > 0 && len(m.queuedReplies) > 0 {
m.data = m.queuedReplies[0]
m.queuedReplies = m.queuedReplies[1:]
}
return
}
func (m *mockConn) Close() error {
m.closed = true
return nil
}
func (m *mockConn) LocalAddr() net.Addr {
return m.laddr
}
func (m *mockConn) RemoteAddr() net.Addr {
return m.raddr
}
func (m *mockConn) SetDeadline(t time.Time) error {
return nil
}
func (m *mockConn) SetReadDeadline(t time.Time) error {
return nil
}
func (m *mockConn) SetWriteDeadline(t time.Time) error {
return nil
}
// make sure mockConn implements the net.Conn interface
var _ net.Conn = new(mockConn)
func newRWMockConn(sequence uint8) (*mockConn, *mysqlConn) {
conn := new(mockConn)
mc := &mysqlConn{
buf: newBuffer(conn),
cfg: NewConfig(),
netConn: conn,
closech: make(chan struct{}),
maxAllowedPacket: defaultMaxAllowedPacket,
sequence: sequence,
}
return conn, mc
}
func TestReadPacketSingleByte(t *testing.T) {
conn := new(mockConn)
mc := &mysqlConn{
buf: newBuffer(conn),
}
conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff}
conn.maxReads = 1
packet, err := mc.readPacket()
if err != nil {
t.Fatal(err)
}
if len(packet) != 1 {
t.Fatalf("unexpected packet length: expected %d, got %d", 1, len(packet))
}
if packet[0] != 0xff {
t.Fatalf("unexpected packet content: expected %x, got %x", 0xff, packet[0])
}
}
func TestReadPacketWrongSequenceID(t *testing.T) {
conn := new(mockConn)
mc := &mysqlConn{
buf: newBuffer(conn),
}
// too low sequence id
conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff}
conn.maxReads = 1
mc.sequence = 1
_, err := mc.readPacket()
if err != ErrPktSync {
t.Errorf("expected ErrPktSync, got %v", err)
}
// reset
conn.reads = 0
mc.sequence = 0
mc.buf = newBuffer(conn)
// too high sequence id
conn.data = []byte{0x01, 0x00, 0x00, 0x42, 0xff}
_, err = mc.readPacket()
if err != ErrPktSyncMul {
t.Errorf("expected ErrPktSyncMul, got %v", err)
}
}
func TestReadPacketSplit(t *testing.T) {
conn := new(mockConn)
mc := &mysqlConn{
buf: newBuffer(conn),
}
data := make([]byte, maxPacketSize*2+4*3)
const pkt2ofs = maxPacketSize + 4
const pkt3ofs = 2 * (maxPacketSize + 4)
// case 1: payload has length maxPacketSize
data = data[:pkt2ofs+4]
// 1st packet has maxPacketSize length and sequence id 0
// ff ff ff 00 ...
data[0] = 0xff
data[1] = 0xff
data[2] = 0xff
// mark the payload start and end of 1st packet so that we can check if the
// content was correctly appended
data[4] = 0x11
data[maxPacketSize+3] = 0x22
// 2nd packet has payload length 0 and squence id 1
// 00 00 00 01
data[pkt2ofs+3] = 0x01
conn.data = data
conn.maxReads = 3
packet, err := mc.readPacket()
if err != nil {
t.Fatal(err)
}
if len(packet) != maxPacketSize {
t.Fatalf("unexpected packet length: expected %d, got %d", maxPacketSize, len(packet))
}
if packet[0] != 0x11 {
t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0])
}
if packet[maxPacketSize-1] != 0x22 {
t.Fatalf("unexpected payload end: expected %x, got %x", 0x22, packet[maxPacketSize-1])
}
// case 2: payload has length which is a multiple of maxPacketSize
data = data[:cap(data)]
// 2nd packet now has maxPacketSize length
data[pkt2ofs] = 0xff
data[pkt2ofs+1] = 0xff
data[pkt2ofs+2] = 0xff
// mark the payload start and end of the 2nd packet
data[pkt2ofs+4] = 0x33
data[pkt2ofs+maxPacketSize+3] = 0x44
// 3rd packet has payload length 0 and squence id 2
// 00 00 00 02
data[pkt3ofs+3] = 0x02
conn.data = data
conn.reads = 0
conn.maxReads = 5
mc.sequence = 0
packet, err = mc.readPacket()
if err != nil {
t.Fatal(err)
}
if len(packet) != 2*maxPacketSize {
t.Fatalf("unexpected packet length: expected %d, got %d", 2*maxPacketSize, len(packet))
}
if packet[0] != 0x11 {
t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0])
}
if packet[2*maxPacketSize-1] != 0x44 {
t.Fatalf("unexpected payload end: expected %x, got %x", 0x44, packet[2*maxPacketSize-1])
}
// case 3: payload has a length larger maxPacketSize, which is not an exact
// multiple of it
data = data[:pkt2ofs+4+42]
data[pkt2ofs] = 0x2a
data[pkt2ofs+1] = 0x00
data[pkt2ofs+2] = 0x00
data[pkt2ofs+4+41] = 0x44
conn.data = data
conn.reads = 0
conn.maxReads = 4
mc.sequence = 0
packet, err = mc.readPacket()
if err != nil {
t.Fatal(err)
}
if len(packet) != maxPacketSize+42 {
t.Fatalf("unexpected packet length: expected %d, got %d", maxPacketSize+42, len(packet))
}
if packet[0] != 0x11 {
t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0])
}
if packet[maxPacketSize+41] != 0x44 {
t.Fatalf("unexpected payload end: expected %x, got %x", 0x44, packet[maxPacketSize+41])
}
}
func TestReadPacketFail(t *testing.T) {
conn := new(mockConn)
mc := &mysqlConn{
buf: newBuffer(conn),
closech: make(chan struct{}),
}
// illegal empty (stand-alone) packet
conn.data = []byte{0x00, 0x00, 0x00, 0x00}
conn.maxReads = 1
_, err := mc.readPacket()
if err != ErrInvalidConn {
t.Errorf("expected ErrInvalidConn, got %v", err)
}
// reset
conn.reads = 0
mc.sequence = 0
mc.buf = newBuffer(conn)
// fail to read header
conn.closed = true
_, err = mc.readPacket()
if err != ErrInvalidConn {
t.Errorf("expected ErrInvalidConn, got %v", err)
}
// reset
conn.closed = false
conn.reads = 0
mc.sequence = 0
mc.buf = newBuffer(conn)
// fail to read body
conn.maxReads = 1
_, err = mc.readPacket()
if err != ErrInvalidConn {
t.Errorf("expected ErrInvalidConn, got %v", err)
}
}
// https://github.com/go-sql-driver/mysql/pull/801
// not-NUL terminated plugin_name in init packet
func TestRegression801(t *testing.T) {
conn := new(mockConn)
mc := &mysqlConn{
buf: newBuffer(conn),
cfg: new(Config),
sequence: 42,
closech: make(chan struct{}),
}
conn.data = []byte{72, 0, 0, 42, 10, 53, 46, 53, 46, 56, 0, 165, 0, 0, 0,
60, 70, 63, 58, 68, 104, 34, 97, 0, 223, 247, 33, 2, 0, 15, 128, 21, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 120, 114, 47, 85, 75, 109, 99, 51, 77,
50, 64, 0, 109, 121, 115, 113, 108, 95, 110, 97, 116, 105, 118, 101, 95,
112, 97, 115, 115, 119, 111, 114, 100}
conn.maxReads = 1
authData, pluginName, err := mc.readHandshakePacket()
if err != nil {
t.Fatalf("got error: %v", err)
}
if pluginName != "mysql_native_password" {
t.Errorf("expected plugin name 'mysql_native_password', got '%s'", pluginName)
}
expectedAuthData := []byte{60, 70, 63, 58, 68, 104, 34, 97, 98, 120, 114,
47, 85, 75, 109, 99, 51, 77, 50, 64}
if !bytes.Equal(authData, expectedAuthData) {
t.Errorf("expected authData '%v', got '%v'", expectedAuthData, authData)
}
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
type mysqlResult struct {
affectedRows int64
insertId int64
}
func (res *mysqlResult) LastInsertId() (int64, error) {
return res.insertId, nil
}
func (res *mysqlResult) RowsAffected() (int64, error) {
return res.affectedRows, nil
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"database/sql/driver"
"io"
"math"
"reflect"
)
type resultSet struct {
columns []mysqlField
columnNames []string
done bool
}
type mysqlRows struct {
mc *mysqlConn
rs resultSet
finish func()
}
type binaryRows struct {
mysqlRows
}
type textRows struct {
mysqlRows
}
func (rows *mysqlRows) Columns() []string {
if rows.rs.columnNames != nil {
return rows.rs.columnNames
}
columns := make([]string, len(rows.rs.columns))
if rows.mc != nil && rows.mc.cfg.ColumnsWithAlias {
for i := range columns {
if tableName := rows.rs.columns[i].tableName; len(tableName) > 0 {
columns[i] = tableName + "." + rows.rs.columns[i].name
} else {
columns[i] = rows.rs.columns[i].name
}
}
} else {
for i := range columns {
columns[i] = rows.rs.columns[i].name
}
}
rows.rs.columnNames = columns
return columns
}
func (rows *mysqlRows) ColumnTypeDatabaseTypeName(i int) string {
return rows.rs.columns[i].typeDatabaseName()
}
// func (rows *mysqlRows) ColumnTypeLength(i int) (length int64, ok bool) {
// return int64(rows.rs.columns[i].length), true
// }
func (rows *mysqlRows) ColumnTypeNullable(i int) (nullable, ok bool) {
return rows.rs.columns[i].flags&flagNotNULL == 0, true
}
func (rows *mysqlRows) ColumnTypePrecisionScale(i int) (int64, int64, bool) {
column := rows.rs.columns[i]
decimals := int64(column.decimals)
switch column.fieldType {
case fieldTypeDecimal, fieldTypeNewDecimal:
if decimals > 0 {
return int64(column.length) - 2, decimals, true
}
return int64(column.length) - 1, decimals, true
case fieldTypeTimestamp, fieldTypeDateTime, fieldTypeTime:
return decimals, decimals, true
case fieldTypeFloat, fieldTypeDouble:
if decimals == 0x1f {
return math.MaxInt64, math.MaxInt64, true
}
return math.MaxInt64, decimals, true
}
return 0, 0, false
}
func (rows *mysqlRows) ColumnTypeScanType(i int) reflect.Type {
return rows.rs.columns[i].scanType()
}
func (rows *mysqlRows) Close() (err error) {
if f := rows.finish; f != nil {
f()
rows.finish = nil
}
mc := rows.mc
if mc == nil {
return nil
}
if err := mc.error(); err != nil {
return err
}
// flip the buffer for this connection if we need to drain it.
// note that for a successful query (i.e. one where rows.next()
// has been called until it returns false), `rows.mc` will be nil
// by the time the user calls `(*Rows).Close`, so we won't reach this
// see: https://github.com/golang/go/commit/651ddbdb5056ded455f47f9c494c67b389622a47
mc.buf.flip()
// Remove unread packets from stream
if !rows.rs.done {
err = mc.readUntilEOF()
}
if err == nil {
if err = mc.discardResults(); err != nil {
return err
}
}
rows.mc = nil
return err
}
func (rows *mysqlRows) HasNextResultSet() (b bool) {
if rows.mc == nil {
return false
}
return rows.mc.status&statusMoreResultsExists != 0
}
func (rows *mysqlRows) nextResultSet() (int, error) {
if rows.mc == nil {
return 0, io.EOF
}
if err := rows.mc.error(); err != nil {
return 0, err
}
// Remove unread packets from stream
if !rows.rs.done {
if err := rows.mc.readUntilEOF(); err != nil {
return 0, err
}
rows.rs.done = true
}
if !rows.HasNextResultSet() {
rows.mc = nil
return 0, io.EOF
}
rows.rs = resultSet{}
return rows.mc.readResultSetHeaderPacket()
}
func (rows *mysqlRows) nextNotEmptyResultSet() (int, error) {
for {
resLen, err := rows.nextResultSet()
if err != nil {
return 0, err
}
if resLen > 0 {
return resLen, nil
}
rows.rs.done = true
}
}
func (rows *binaryRows) NextResultSet() error {
resLen, err := rows.nextNotEmptyResultSet()
if err != nil {
return err
}
rows.rs.columns, err = rows.mc.readColumns(resLen)
return err
}
func (rows *binaryRows) Next(dest []driver.Value) error {
if mc := rows.mc; mc != nil {
if err := mc.error(); err != nil {
return err
}
// Fetch next row from stream
return rows.readRow(dest)
}
return io.EOF
}
func (rows *textRows) NextResultSet() (err error) {
resLen, err := rows.nextNotEmptyResultSet()
if err != nil {
return err
}
rows.rs.columns, err = rows.mc.readColumns(resLen)
return err
}
func (rows *textRows) Next(dest []driver.Value) error {
if mc := rows.mc; mc != nil {
if err := mc.error(); err != nil {
return err
}
// Fetch next row from stream
return rows.readRow(dest)
}
return io.EOF
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"database/sql/driver"
"encoding/json"
"fmt"
"io"
"reflect"
)
type mysqlStmt struct {
mc *mysqlConn
id uint32
paramCount int
}
func (stmt *mysqlStmt) Close() error {
if stmt.mc == nil || stmt.mc.closed.IsSet() {
// driver.Stmt.Close can be called more than once, thus this function
// has to be idempotent.
// See also Issue #450 and golang/go#16019.
//errLog.Print(ErrInvalidConn)
return driver.ErrBadConn
}
err := stmt.mc.writeCommandPacketUint32(comStmtClose, stmt.id)
stmt.mc = nil
return err
}
func (stmt *mysqlStmt) NumInput() int {
return stmt.paramCount
}
func (stmt *mysqlStmt) ColumnConverter(idx int) driver.ValueConverter {
return converter{}
}
func (stmt *mysqlStmt) CheckNamedValue(nv *driver.NamedValue) (err error) {
nv.Value, err = converter{}.ConvertValue(nv.Value)
return
}
func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) {
if stmt.mc.closed.IsSet() {
errLog.Print(ErrInvalidConn)
return nil, driver.ErrBadConn
}
// Send command
err := stmt.writeExecutePacket(args)
if err != nil {
return nil, stmt.mc.markBadConn(err)
}
mc := stmt.mc
mc.affectedRows = 0
mc.insertId = 0
// Read Result
resLen, err := mc.readResultSetHeaderPacket()
if err != nil {
return nil, err
}
if resLen > 0 {
// Columns
if err = mc.readUntilEOF(); err != nil {
return nil, err
}
// Rows
if err := mc.readUntilEOF(); err != nil {
return nil, err
}
}
if err := mc.discardResults(); err != nil {
return nil, err
}
return &mysqlResult{
affectedRows: int64(mc.affectedRows),
insertId: int64(mc.insertId),
}, nil
}
func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) {
return stmt.query(args)
}
func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) {
if stmt.mc.closed.IsSet() {
errLog.Print(ErrInvalidConn)
return nil, driver.ErrBadConn
}
// Send command
err := stmt.writeExecutePacket(args)
if err != nil {
return nil, stmt.mc.markBadConn(err)
}
mc := stmt.mc
// Read Result
resLen, err := mc.readResultSetHeaderPacket()
if err != nil {
return nil, err
}
rows := new(binaryRows)
if resLen > 0 {
rows.mc = mc
rows.rs.columns, err = mc.readColumns(resLen)
} else {
rows.rs.done = true
switch err := rows.NextResultSet(); err {
case nil, io.EOF:
return rows, nil
default:
return nil, err
}
}
return rows, err
}
var jsonType = reflect.TypeOf(json.RawMessage{})
type converter struct{}
// ConvertValue mirrors the reference/default converter in database/sql/driver
// with _one_ exception. We support uint64 with their high bit and the default
// implementation does not. This function should be kept in sync with
// database/sql/driver defaultConverter.ConvertValue() except for that
// deliberate difference.
func (c converter) ConvertValue(v interface{}) (driver.Value, error) {
if driver.IsValue(v) {
return v, nil
}
if vr, ok := v.(driver.Valuer); ok {
sv, err := callValuerValue(vr)
if err != nil {
return nil, err
}
if driver.IsValue(sv) {
return sv, nil
}
// A value returend from the Valuer interface can be "a type handled by
// a database driver's NamedValueChecker interface" so we should accept
// uint64 here as well.
if u, ok := sv.(uint64); ok {
return u, nil
}
return nil, fmt.Errorf("non-Value type %T returned from Value", sv)
}
rv := reflect.ValueOf(v)
switch rv.Kind() {
case reflect.Ptr:
// indirect pointers
if rv.IsNil() {
return nil, nil
} else {
return c.ConvertValue(rv.Elem().Interface())
}
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return rv.Int(), nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return rv.Uint(), nil
case reflect.Float32, reflect.Float64:
return rv.Float(), nil
case reflect.Bool:
return rv.Bool(), nil
case reflect.Slice:
switch t := rv.Type(); {
case t == jsonType:
return v, nil
case t.Elem().Kind() == reflect.Uint8:
return rv.Bytes(), nil
default:
return nil, fmt.Errorf("unsupported type %T, a slice of %s", v, t.Elem().Kind())
}
case reflect.String:
return rv.String(), nil
}
return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind())
}
var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem()
// callValuerValue returns vr.Value(), with one exception:
// If vr.Value is an auto-generated method on a pointer type and the
// pointer is nil, it would panic at runtime in the panicwrap
// method. Treat it like nil instead.
//
// This is so people can implement driver.Value on value types and
// still use nil pointers to those types to mean nil/NULL, just like
// string/*string.
//
// This is an exact copy of the same-named unexported function from the
// database/sql package.
func callValuerValue(vr driver.Valuer) (v driver.Value, err error) {
if rv := reflect.ValueOf(vr); rv.Kind() == reflect.Ptr &&
rv.IsNil() &&
rv.Type().Elem().Implements(valuerReflectType) {
return nil, nil
}
return vr.Value()
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2017 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
import (
"bytes"
"database/sql/driver"
"encoding/json"
"testing"
)
func TestConvertDerivedString(t *testing.T) {
type derived string
output, err := converter{}.ConvertValue(derived("value"))
if err != nil {
t.Fatal("Derived string type not convertible", err)
}
if output != "value" {
t.Fatalf("Derived string type not converted, got %#v %T", output, output)
}
}
func TestConvertDerivedByteSlice(t *testing.T) {
type derived []uint8
output, err := converter{}.ConvertValue(derived("value"))
if err != nil {
t.Fatal("Byte slice not convertible", err)
}
if bytes.Compare(output.([]byte), []byte("value")) != 0 {
t.Fatalf("Byte slice not converted, got %#v %T", output, output)
}
}
func TestConvertDerivedUnsupportedSlice(t *testing.T) {
type derived []int
_, err := converter{}.ConvertValue(derived{1})
if err == nil || err.Error() != "unsupported type mysql.derived, a slice of int" {
t.Fatal("Unexpected error", err)
}
}
func TestConvertDerivedBool(t *testing.T) {
type derived bool
output, err := converter{}.ConvertValue(derived(true))
if err != nil {
t.Fatal("Derived bool type not convertible", err)
}
if output != true {
t.Fatalf("Derived bool type not converted, got %#v %T", output, output)
}
}
func TestConvertPointer(t *testing.T) {
str := "value"
output, err := converter{}.ConvertValue(&str)
if err != nil {
t.Fatal("Pointer type not convertible", err)
}
if output != "value" {
t.Fatalf("Pointer type not converted, got %#v %T", output, output)
}
}
func TestConvertSignedIntegers(t *testing.T) {
values := []interface{}{
int8(-42),
int16(-42),
int32(-42),
int64(-42),
int(-42),
}
for _, value := range values {
output, err := converter{}.ConvertValue(value)
if err != nil {
t.Fatalf("%T type not convertible %s", value, err)
}
if output != int64(-42) {
t.Fatalf("%T type not converted, got %#v %T", value, output, output)
}
}
}
type myUint64 struct {
value uint64
}
func (u myUint64) Value() (driver.Value, error) {
return u.value, nil
}
func TestConvertUnsignedIntegers(t *testing.T) {
values := []interface{}{
uint8(42),
uint16(42),
uint32(42),
uint64(42),
uint(42),
myUint64{uint64(42)},
}
for _, value := range values {
output, err := converter{}.ConvertValue(value)
if err != nil {
t.Fatalf("%T type not convertible %s", value, err)
}
if output != uint64(42) {
t.Fatalf("%T type not converted, got %#v %T", value, output, output)
}
}
output, err := converter{}.ConvertValue(^uint64(0))
if err != nil {
t.Fatal("uint64 high-bit not convertible", err)
}
if output != ^uint64(0) {
t.Fatalf("uint64 high-bit converted, got %#v %T", output, output)
}
}
func TestConvertJSON(t *testing.T) {
raw := json.RawMessage("{}")
out, err := converter{}.ConvertValue(raw)
if err != nil {
t.Fatal("json.RawMessage was failed in convert", err)
}
if _, ok := out.(json.RawMessage); !ok {
t.Fatalf("json.RawMessage converted, got %#v %T", out, out)
}
}
// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package mysql
type mysqlTx struct {
mc *mysqlConn
}
func (tx *mysqlTx) Commit() (err error) {
if tx.mc == nil || tx.mc.closed.IsSet() {
return ErrInvalidConn
}
err = tx.mc.exec("COMMIT")
tx.mc = nil
return
}
func (tx *mysqlTx) Rollback() (err error) {
if tx.mc == nil || tx.mc.closed.IsSet() {
return ErrInvalidConn
}
err = tx.mc.exec("ROLLBACK")
tx.mc = nil
return
}
Docs: https://godoc.org/github.com/josharian/intern
For a blog post introducing string interning, see: https://commaok.xyz/post/intern-strings/
License: MIT
module github.com/josharian/intern
go 1.5
// Package intern interns strings.
// Interning is best effort only.
// Interned strings may be removed automatically
// at any time without notification.
// All functions may be called concurrently
// with themselves and each other.
package intern
import "sync"
var (
pool sync.Pool = sync.Pool{
New: func() interface{} {
return make(map[string]string)
},
}
)
// String returns s, interned.
func String(s string) string {
m := pool.Get().(map[string]string)
c, ok := m[s]
if ok {
pool.Put(m)
return c
}
m[s] = s
pool.Put(m)
return s
}
// Bytes returns b converted to a string, interned.
func Bytes(b []byte) string {
m := pool.Get().(map[string]string)
c, ok := m[string(b)]
if ok {
pool.Put(m)
return c
}
s := string(b)
m[s] = s
pool.Put(m)
return s
}
// +build !race
// When -race is enabled, sync.Pool is a no-op,
// which will cause these tests to fail
// and these benchmarks to be misleading.
package intern
import (
"bytes"
"reflect"
"testing"
"unsafe"
)
func TestString(t *testing.T) {
s := "abcde"
sub := String(s[1:4])
interned := String("bcd")
want := (*reflect.StringHeader)(unsafe.Pointer(&sub)).Data
got := (*reflect.StringHeader)(unsafe.Pointer(&interned)).Data
if want != got {
t.Errorf("failed to intern string")
}
}
func TestBytes(t *testing.T) {
s := bytes.Repeat([]byte("abc"), 100)
n := testing.AllocsPerRun(100, func() {
for i := 0; i < 100; i++ {
_ = Bytes(s[i*len("abc") : (i+1)*len("abc")])
}
})
if n > 0 {
t.Errorf("Bytes allocated %d, want 0", int(n))
}
}
func BenchmarkString(b *testing.B) {
in := "hello brad"
b.ReportAllocs()
b.SetBytes(int64(len(in)))
b.RunParallel(func(pb *testing.PB) {
var s string
for pb.Next() {
s = String(in[1:5])
}
_ = s
})
}
func BenchmarkBytes(b *testing.B) {
in := []byte("hello brad")
b.ReportAllocs()
b.SetBytes(int64(len(in)))
b.RunParallel(func(pb *testing.PB) {
var s string
for pb.Next() {
s = Bytes(in[1:5])
}
_ = s
})
}
MIT License
Copyright (c) 2019 Josh Bleecher Snyder
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
# cedar-go [![GoDoc](https://godoc.org/github.com/adamzy/cedar-go?status.svg)](https://godoc.org/github.com/adamzy/cedar-go)
Package `cedar-go` implementes double-array trie.
It is a [Golang](https://golang.org/) port of [cedar](http://www.tkl.iis.u-tokyo.ac.jp/~ynaga/cedar) which is written in C++ by Naoki Yoshinaga. `cedar-go` currently implements the `reduced` verion of cedar.
This package is not thread safe if there is one goroutine doing insertions or deletions.
## Install
```
go get github.com/adamzy/cedar-go
```
## Usage
```go
package main
import (
"fmt"
"github.com/adamzy/cedar-go"
)
func main() {
// create a new cedar trie.
trie := cedar.New()
// a helper function to print the id-key-value triple given trie node id
printIdKeyValue := func(id int) {
// the key of node `id`.
key, _ := trie.Key(id)
// the value of node `id`.
value, _ := trie.Value(id)
fmt.Printf("%d\t%s:%v\n", id, key, value)
}
// Insert key-value pairs.
// The order of insertion is not important.
trie.Insert([]byte("How many"), 0)
trie.Insert([]byte("How many loved"), 1)
trie.Insert([]byte("How many loved your moments"), 2)
trie.Insert([]byte("How many loved your moments of glad grace"), 3)
trie.Insert([]byte("姑苏"), 4)
trie.Insert([]byte("姑苏城外"), 5)
trie.Insert([]byte("姑苏城外寒山寺"), 6)
// Get the associated value of a key directly.
value, _ := trie.Get([]byte("How many loved your moments of glad grace"))
fmt.Println(value)
// Or, jump to the node first,
id, _ := trie.Jump([]byte("How many loved your moments"), 0)
// then get the key and the value
printIdKeyValue(id)
fmt.Println("\nPrefixMatch\nid\tkey:value")
for _, id := range trie.PrefixMatch([]byte("How many loved your moments of glad grace"), 0) {
printIdKeyValue(id)
}
fmt.Println("\nPrefixPredict\nid\tkey:value")
for _, id := range trie.PrefixPredict([]byte("姑苏"), 0) {
printIdKeyValue(id)
}
}
```
will produce
```
3
281 How many loved your moments:2
PrefixMatch
id key:value
262 How many:0
268 How many loved:1
281 How many loved your moments:2
296 How many loved your moments of glad grace:3
PrefixPredict
id key:value
303 姑苏:4
309 姑苏城外:5
318 姑苏城外寒山寺:6
```
package cedar
// Status reports the following statistics of the cedar:
// keys: number of keys that are in the cedar,
// nodes: number of trie nodes (slots in the base array) has been taken,
// size: the size of the base array used by the cedar,
// capacity: the capicity of the base array used by the cedar.
func (da *Cedar) Status() (keys, nodes, size, capacity int) {
for i := 0; i < da.Size; i++ {
n := da.Array[i]
if n.Check >= 0 {
nodes++
if n.Value >= 0 {
keys++
}
}
}
return keys, nodes, da.Size, da.Capacity
}
// Jump travels from a node `from` to another node `to` by following the path `path`.
// For example, if the following keys were inserted:
// id key
// 19 abc
// 23 ab
// 37 abcd
// then
// Jump([]byte("ab"), 0) = 23, nil // reach "ab" from root
// Jump([]byte("c"), 23) = 19, nil // reach "abc" from "ab"
// Jump([]byte("cd"), 23) = 37, nil // reach "abcd" from "ab"
func (da *Cedar) Jump(path []byte, from int) (to int, err error) {
for _, b := range path {
if da.Array[from].Value >= 0 {
return from, ErrNoPath
}
to = da.Array[from].base() ^ int(b)
if da.Array[to].Check != from {
return from, ErrNoPath
}
from = to
}
return to, nil
}
// Key returns the key of the node with the given `id`.
// It will return ErrNoPath, if the node does not exist.
func (da *Cedar) Key(id int) (key []byte, err error) {
for id > 0 {
from := da.Array[id].Check
if from < 0 {
return nil, ErrNoPath
}
if char := byte(da.Array[from].base() ^ id); char != 0 {
key = append(key, char)
}
id = from
}
if id != 0 || len(key) == 0 {
return nil, ErrInvalidKey
}
for i := 0; i < len(key)/2; i++ {
key[i], key[len(key)-i-1] = key[len(key)-i-1], key[i]
}
return key, nil
}
// Value returns the value of the node with the given `id`.
// It will return ErrNoValue, if the node does not have a value.
func (da *Cedar) Value(id int) (value int, err error) {
value = da.Array[id].Value
if value >= 0 {
return value, nil
}
to := da.Array[id].base()
if da.Array[to].Check == id && da.Array[to].Value >= 0 {
return da.Array[to].Value, nil
}
return 0, ErrNoValue
}
// Insert adds a key-value pair into the cedar.
// It will return ErrInvalidValue, if value < 0 or >= ValueLimit.
func (da *Cedar) Insert(key []byte, value int) error {
if value < 0 || value >= ValueLimit {
return ErrInvalidValue
}
p := da.get(key, 0, 0)
*p = value
return nil
}
// Update increases the value associated with the `key`.
// The `key` will be inserted if it is not in the cedar.
// It will return ErrInvalidValue, if the updated value < 0 or >= ValueLimit.
func (da *Cedar) Update(key []byte, value int) error {
p := da.get(key, 0, 0)
// key was not inserted
if *p == ValueLimit {
*p = value
return nil
}
// key was inserted before
if *p+value < 0 || *p+value >= ValueLimit {
return ErrInvalidValue
}
*p += value
return nil
}
// Delete removes a key-value pair from the cedar.
// It will return ErrNoPath, if the key has not been added.
func (da *Cedar) Delete(key []byte) error {
// if the path does not exist, or the end is not a leaf, nothing to delete
to, err := da.Jump(key, 0)
if err != nil {
return ErrNoPath
}
if da.Array[to].Value < 0 {
base := da.Array[to].base()
if da.Array[base].Check == to {
to = base
}
}
for to > 0 {
from := da.Array[to].Check
base := da.Array[from].base()
label := byte(to ^ base)
// if `to` has sibling, remove `to` from the sibling list, then stop
if da.Ninfos[to].Sibling != 0 || da.Ninfos[from].Child != label {
// delete the label from the child ring first
da.popSibling(from, base, label)
// then release the current node `to` to the empty node ring
da.pushEnode(to)
break
}
// otherwise, just release the current node `to` to the empty node ring
da.pushEnode(to)
// then check its parent node
to = from
}
return nil
}
// Get returns the value associated with the given `key`.
// It is equivalent to
// id, err1 = Jump(key)
// value, err2 = Value(id)
// Thus, it may return ErrNoPath or ErrNoValue,
func (da *Cedar) Get(key []byte) (value int, err error) {
to, err := da.Jump(key, 0)
if err != nil {
return 0, err
}
return da.Value(to)
}
// PrefixMatch returns a list of at most `num` nodes which match the prefix of the key.
// If `num` is 0, it returns all matches.
// For example, if the following keys were inserted:
// id key
// 19 abc
// 23 ab
// 37 abcd
// then
// PrefixMatch([]byte("abc"), 1) = [ 23 ] // match ["ab"]
// PrefixMatch([]byte("abcd"), 0) = [ 23, 19, 37] // match ["ab", "abc", "abcd"]
func (da *Cedar) PrefixMatch(key []byte, num int) (ids []int) {
for from, i := 0, 0; i < len(key); i++ {
to, err := da.Jump(key[i:i+1], from)
if err != nil {
break
}
if _, err := da.Value(to); err == nil {
ids = append(ids, to)
num--
if num == 0 {
return
}
}
from = to
}
return
}
// PrefixPredict returns a list of at most `num` nodes which has the key as their prefix.
// These nodes are ordered by their keys.
// If `num` is 0, it returns all matches.
// For example, if the following keys were inserted:
// id key
// 19 abc
// 23 ab
// 37 abcd
// then
// PrefixPredict([]byte("ab"), 2) = [ 23, 19 ] // predict ["ab", "abc"]
// PrefixPredict([]byte("ab"), 0) = [ 23, 19, 37 ] // predict ["ab", "abc", "abcd"]
func (da *Cedar) PrefixPredict(key []byte, num int) (ids []int) {
root, err := da.Jump(key, 0)
if err != nil {
return
}
for from, err := da.begin(root); err == nil; from, err = da.next(from, root) {
ids = append(ids, from)
num--
if num == 0 {
return
}
}
return
}
func (da *Cedar) begin(from int) (to int, err error) {
for c := da.Ninfos[from].Child; c != 0; {
to = da.Array[from].base() ^ int(c)
c = da.Ninfos[to].Child
from = to
}
if da.Array[from].base() > 0 {
return da.Array[from].base(), nil
}
return from, nil
}
func (da *Cedar) next(from int, root int) (to int, err error) {
c := da.Ninfos[from].Sibling
for c == 0 && from != root && da.Array[from].Check >= 0 {
from = da.Array[from].Check
c = da.Ninfos[from].Sibling
}
if from == root {
return 0, ErrNoPath
}
from = da.Array[da.Array[from].Check].base() ^ int(c)
return da.begin(from)
}
package cedar
const ValueLimit = int(^uint(0) >> 1)
type node struct {
Value int
Check int
}
func (n *node) base() int { return -(n.Value + 1) }
type ninfo struct {
Sibling, Child byte
}
type block struct {
Prev, Next, Num, Reject, Trial, Ehead int
}
func (b *block) init() {
b.Num = 256
b.Reject = 257
}
type Cedar struct {
*cedar
}
type cedar struct {
Array []node
Ninfos []ninfo
Blocks []block
Reject [257]int
BheadF int
BheadC int
BheadO int
Capacity int
Size int
Ordered bool
MaxTrial int
}
func New() *Cedar {
da := cedar{
Array: make([]node, 256),
Ninfos: make([]ninfo, 256),
Blocks: make([]block, 1),
Capacity: 256,
Size: 256,
Ordered: true,
MaxTrial: 1,
}
da.Array[0] = node{-2, 0}
for i := 1; i < 256; i++ {
da.Array[i] = node{-(i - 1), -(i + 1)}
}
da.Array[1].Value = -255
da.Array[255].Check = -1
da.Blocks[0].Ehead = 1
da.Blocks[0].init()
for i := 0; i <= 256; i++ {
da.Reject[i] = i + 1
}
return &Cedar{&da}
}
// Get value by key, insert the key if not exist
func (da *cedar) get(key []byte, from, pos int) *int {
for ; pos < len(key); pos++ {
if value := da.Array[from].Value; value >= 0 && value != ValueLimit {
to := da.follow(from, 0)
da.Array[to].Value = value
}
from = da.follow(from, key[pos])
}
to := from
if da.Array[from].Value < 0 {
to = da.follow(from, 0)
}
return &da.Array[to].Value
}
func (da *cedar) follow(from int, label byte) int {
base := da.Array[from].base()
to := base ^ int(label)
if base < 0 || da.Array[to].Check < 0 {
hasChild := false
if base >= 0 {
hasChild = (da.Array[base^int(da.Ninfos[from].Child)].Check == from)
}
to = da.popEnode(base, label, from)
da.pushSibling(from, to^int(label), label, hasChild)
} else if da.Array[to].Check != from {
to = da.resolve(from, base, label)
} else if da.Array[to].Check == from {
} else {
panic("cedar: internal error, should not be here")
}
return to
}
func (da *cedar) popBlock(bi int, head_in *int, last bool) {
if last {
*head_in = 0
} else {
b := &da.Blocks[bi]
da.Blocks[b.Prev].Next = b.Next
da.Blocks[b.Next].Prev = b.Prev
if bi == *head_in {
*head_in = b.Next
}
}
}
func (da *cedar) pushBlock(bi int, head_out *int, empty bool) {
b := &da.Blocks[bi]
if empty {
*head_out, b.Prev, b.Next = bi, bi, bi
} else {
tail_out := &da.Blocks[*head_out].Prev
b.Prev = *tail_out
b.Next = *head_out
*head_out, *tail_out, da.Blocks[*tail_out].Next = bi, bi, bi
}
}
func (da *cedar) addBlock() int {
if da.Size == da.Capacity {
da.Capacity *= 2
oldArray := da.Array
da.Array = make([]node, da.Capacity)
copy(da.Array, oldArray)
oldNinfo := da.Ninfos
da.Ninfos = make([]ninfo, da.Capacity)
copy(da.Ninfos, oldNinfo)
oldBlock := da.Blocks
da.Blocks = make([]block, da.Capacity>>8)
copy(da.Blocks, oldBlock)
}
da.Blocks[da.Size>>8].init()
da.Blocks[da.Size>>8].Ehead = da.Size
da.Array[da.Size] = node{-(da.Size + 255), -(da.Size + 1)}
for i := da.Size + 1; i < da.Size+255; i++ {
da.Array[i] = node{-(i - 1), -(i + 1)}
}
da.Array[da.Size+255] = node{-(da.Size + 254), -da.Size}
da.pushBlock(da.Size>>8, &da.BheadO, da.BheadO == 0)
da.Size += 256
return da.Size>>8 - 1
}
func (da *cedar) transferBlock(bi int, head_in, head_out *int) {
da.popBlock(bi, head_in, bi == da.Blocks[bi].Next)
da.pushBlock(bi, head_out, *head_out == 0 && da.Blocks[bi].Num != 0)
}
func (da *cedar) popEnode(base int, label byte, from int) int {
e := base ^ int(label)
if base < 0 {
e = da.findPlace()
}
bi := e >> 8
n := &da.Array[e]
b := &da.Blocks[bi]
b.Num--
if b.Num == 0 {
if bi != 0 {
da.transferBlock(bi, &da.BheadC, &da.BheadF)
}
} else {
da.Array[-n.Value].Check = n.Check
da.Array[-n.Check].Value = n.Value
if e == b.Ehead {
b.Ehead = -n.Check
}
if bi != 0 && b.Num == 1 && b.Trial != da.MaxTrial {
da.transferBlock(bi, &da.BheadO, &da.BheadC)
}
}
n.Value = ValueLimit
n.Check = from
if base < 0 {
da.Array[from].Value = -(e ^ int(label)) - 1
}
return e
}
func (da *cedar) pushEnode(e int) {
bi := e >> 8
b := &da.Blocks[bi]
b.Num++
if b.Num == 1 {
b.Ehead = e
da.Array[e] = node{-e, -e}
if bi != 0 {
da.transferBlock(bi, &da.BheadF, &da.BheadC)
}
} else {
prev := b.Ehead
next := -da.Array[prev].Check
da.Array[e] = node{-prev, -next}
da.Array[prev].Check = -e
da.Array[next].Value = -e
if b.Num == 2 || b.Trial == da.MaxTrial {
if bi != 0 {
da.transferBlock(bi, &da.BheadC, &da.BheadO)
}
}
b.Trial = 0
}
if b.Reject < da.Reject[b.Num] {
b.Reject = da.Reject[b.Num]
}
da.Ninfos[e] = ninfo{}
}
// hasChild: wherether the `from` node has children
func (da *cedar) pushSibling(from, base int, label byte, hasChild bool) {
c := &da.Ninfos[from].Child
keepOrder := *c == 0
if da.Ordered {
keepOrder = label > *c
}
if hasChild && keepOrder {
c = &da.Ninfos[base^int(*c)].Sibling
for da.Ordered && *c != 0 && *c < label {
c = &da.Ninfos[base^int(*c)].Sibling
}
}
da.Ninfos[base^int(label)].Sibling = *c
*c = label
}
func (da *cedar) popSibling(from, base int, label byte) {
c := &da.Ninfos[from].Child
for *c != label {
c = &da.Ninfos[base^int(*c)].Sibling
}
*c = da.Ninfos[base^int(*c)].Sibling
}
func (da *cedar) consult(base_n, base_p int, c_n, c_p byte) bool {
c_n = da.Ninfos[base_n^int(c_n)].Sibling
c_p = da.Ninfos[base_p^int(c_p)].Sibling
for c_n != 0 && c_p != 0 {
c_n = da.Ninfos[base_n^int(c_n)].Sibling
c_p = da.Ninfos[base_p^int(c_p)].Sibling
}
return c_p != 0
}
func (da *cedar) setChild(base int, c byte, label byte, flag bool) []byte {
child := make([]byte, 0, 257)
if c == 0 {
child = append(child, c)
c = da.Ninfos[base^int(c)].Sibling
}
if da.Ordered {
for c != 0 && c <= label {
child = append(child, c)
c = da.Ninfos[base^int(c)].Sibling
}
}
if flag {
child = append(child, label)
}
for c != 0 {
child = append(child, c)
c = da.Ninfos[base^int(c)].Sibling
}
return child
}
func (da *cedar) findPlace() int {
if da.BheadC != 0 {
return da.Blocks[da.BheadC].Ehead
}
if da.BheadO != 0 {
return da.Blocks[da.BheadO].Ehead
}
return da.addBlock() << 8
}
func (da *cedar) findPlaces(child []byte) int {
bi := da.BheadO
if bi != 0 {
bz := da.Blocks[da.BheadO].Prev
nc := len(child)
for {
b := &da.Blocks[bi]
if b.Num >= nc && nc < b.Reject {
for e := b.Ehead; ; {
base := e ^ int(child[0])
for i := 0; da.Array[base^int(child[i])].Check < 0; i++ {
if i == len(child)-1 {
b.Ehead = e
return e
}
}
e = -da.Array[e].Check
if e == b.Ehead {
break
}
}
}
b.Reject = nc
if b.Reject < da.Reject[b.Num] {
da.Reject[b.Num] = b.Reject
}
bi_ := b.Next
b.Trial++
if b.Trial == da.MaxTrial {
da.transferBlock(bi, &da.BheadO, &da.BheadC)
}
if bi == bz {
break
}
bi = bi_
}
}
return da.addBlock() << 8
}
func (da *cedar) resolve(from_n, base_n int, label_n byte) int {
to_pn := base_n ^ int(label_n)
from_p := da.Array[to_pn].Check
base_p := da.Array[from_p].base()
flag := da.consult(base_n, base_p, da.Ninfos[from_n].Child, da.Ninfos[from_p].Child)
var children []byte
if flag {
children = da.setChild(base_n, da.Ninfos[from_n].Child, label_n, true)
} else {
children = da.setChild(base_p, da.Ninfos[from_p].Child, 255, false)
}
var base int
if len(children) == 1 {
base = da.findPlace()
} else {
base = da.findPlaces(children)
}
base ^= int(children[0])
var from int
var base_ int
if flag {
from = from_n
base_ = base_n
} else {
from = from_p
base_ = base_p
}
if flag && children[0] == label_n {
da.Ninfos[from].Child = label_n
}
da.Array[from].Value = -base - 1
for i := 0; i < len(children); i++ {
to := da.popEnode(base, children[i], from)
to_ := base_ ^ int(children[i])
if i == len(children)-1 {
da.Ninfos[to].Sibling = 0
} else {
da.Ninfos[to].Sibling = children[i+1]
}
if flag && to_ == to_pn { // new node has no child
continue
}
n := &da.Array[to]
n_ := &da.Array[to_]
n.Value = n_.Value
if n.Value < 0 && children[i] != 0 {
// this node has children, fix their check
c := da.Ninfos[to_].Child
da.Ninfos[to].Child = c
da.Array[n.base()^int(c)].Check = to
c = da.Ninfos[n.base()^int(c)].Sibling
for c != 0 {
da.Array[n.base()^int(c)].Check = to
c = da.Ninfos[n.base()^int(c)].Sibling
}
}
if !flag && to_ == from_n { // parent node moved
from_n = to
}
if !flag && to_ == to_pn {
da.pushSibling(from_n, to_pn^int(label_n), label_n, true)
da.Ninfos[to_].Child = 0
n_.Value = ValueLimit
n_.Check = from_n
} else {
da.pushEnode(to_)
}
}
if flag {
return base ^ int(label_n)
}
return to_pn
}
package cedar
import (
"fmt"
"os"
"testing"
)
var (
cd *Cedar
words = []string{
"a", "aa", "ab", "ac", "abc", "abd",
"abcd", "abde", "abdf", "abcdef", "abcde",
"abcdefghijklmn", "bcd", "b", "xyz",
"中国", "中国北京", "中国上海", "中国广州",
"中华", "中华文明", "中华民族", "中华人民共和国",
"this", "this is", "this is a sentence.",
}
)
func loadTestData() {
if cd != nil {
return
}
cd = New()
// cd.Ordered = false
// add the keys
for i, word := range words {
if err := cd.Insert([]byte(word), i); err != nil {
panic(err)
}
}
for _, word := range words {
if err := cd.Delete([]byte(word)); err != nil {
panic(err)
}
}
for i, word := range words {
if err := cd.Update([]byte(word), i); err != nil {
panic(err)
}
}
// delete some keys
for i := 0; i < len(words); i += 4 {
if err := cd.Delete([]byte(words[i])); err != nil {
panic(err)
}
}
return
}
func TestBasic(t *testing.T) {
loadTestData()
// check the consistency
checkConsistency(cd)
}
func TestSaveAndLoad(t *testing.T) {
loadTestData()
cd.SaveToFile("cedar.gob", "gob")
defer os.Remove("cedar.gob")
daGob := New()
if err := daGob.LoadFromFile("cedar.gob", "gob"); err != nil {
panic(err)
}
checkConsistency(daGob)
cd.SaveToFile("cedar.json", "json")
defer os.Remove("cedar.json")
daJson := New()
if err := daJson.LoadFromFile("cedar.json", "json"); err != nil {
panic(err)
}
checkConsistency(daJson)
}
func TestPrefixMatch(t *testing.T) {
var ids []int
var keys []string
var values []int
ids = cd.PrefixMatch([]byte("abcdefg"), 0)
keys = []string{"ab", "abcd", "abcde", "abcdef"}
values = []int{2, 6, 10, 9}
check(cd, ids, keys, values)
ids = cd.PrefixMatch([]byte("中华人民共和国"), 0)
keys = []string{"中华", "中华人民共和国"}
values = []int{19, 22}
check(cd, ids, keys, values)
ids = cd.PrefixMatch([]byte("this is a sentence."), 0)
keys = []string{"this", "this is a sentence."}
values = []int{23, 25}
check(cd, ids, keys, values)
}
func check(cd *Cedar, ids []int, keys []string, values []int) {
if len(ids) != len(keys) {
panic("wrong prefix match")
}
for i, n := range ids {
key, _ := cd.Key(n)
val, _ := cd.Value(n)
if string(key) != keys[i] || val != values[i] {
panic("wrong prefix match")
}
}
}
func TestOrder(t *testing.T) {
c := New()
c.Insert([]byte("a"), 1)
c.Insert([]byte("b"), 3)
c.Insert([]byte("d"), 6)
c.Insert([]byte("ab"), 2)
c.Insert([]byte("c"), 5)
c.Insert([]byte(""), 0)
c.Insert([]byte("bb"), 4)
ids := c.PrefixPredict([]byte(""), 0)
if len(ids) != 7 {
panic("wrong order")
}
for i, n := range ids {
val, _ := c.Value(n)
if i != val {
panic("wrong order")
}
}
}
func TestPrefixPredict(t *testing.T) {
var ids []int
var keys []string
var values []int
ids = cd.PrefixPredict([]byte("中华"), 0)
keys = []string{"中华", "中华人民共和国", "中华民族"}
values = []int{19, 22, 21}
check(cd, ids, keys, values)
ids = cd.PrefixPredict([]byte("中国"), 0)
keys = []string{"中国", "中国上海", "中国广州"}
values = []int{15, 17, 18}
check(cd, ids, keys, values)
}
func checkConsistency(cd *Cedar) {
for i, word := range words {
id, err := cd.Jump([]byte(word), 0)
if i%4 == 0 {
if err == ErrNoPath {
continue
}
_, err := cd.Value(id)
if err == ErrNoValue {
continue
}
panic("not deleted")
}
key, err := cd.Key(id)
if err != nil {
panic(err)
}
if string(key) != word {
panic("key error")
}
value, err := cd.Value(id)
if err != nil || value != i {
fmt.Println(word, i, value, err)
panic("value error")
}
}
}
package cedar
import (
"bufio"
"fmt"
"io"
"log"
"os"
"testing"
)
type item struct {
key []byte
value int
}
var dict []item
var trie = New()
func loadDict() {
f, err := os.Open("testdata/dict.txt")
if err != nil {
panic("failed to open testdata/dict.txt")
}
defer f.Close()
in := bufio.NewReader(f)
added := make(map[string]struct{})
var key string
var freq int
var pos string
for {
_, err := fmt.Fscanln(in, &key, &freq, &pos)
if err == io.EOF {
break
}
if _, ok := added[string(key)]; !ok {
dict = append(dict, item{[]byte(key), freq})
added[string(key)] = struct{}{}
}
}
}
func TestLargeDict(t *testing.T) {
loadDict()
size := len(dict)
log.Println("dict size:", size)
exist := func(i int) {
item := dict[i]
// fmt.Println(i, string(item.key))
id, err := trie.Jump(item.key, 0)
failIfError(err)
key, err := trie.Key(id)
failIfError(err)
value, err := trie.Value(id)
failIfError(err)
if string(key) != string(item.key) || value != item.value {
v, _ := trie.Get(item.key)
fmt.Println(i, string(key), string(item.key), value, item.value, v)
panic("large dict test fail: no equal")
}
}
notExist := func(i int) {
_, err := trie.Get(dict[i].key)
// fmt.Println(i, err)
if err != ErrNoPath && err != ErrNoValue {
panic("large dict test fail: should not exist")
}
}
checkSize := func(exp int) {
if keys, _, _, _ := trie.Status(); keys != exp {
panic("not correct status")
}
}
// Insert the first half of the dict.
for i := 0; i < size/2; i++ {
item := dict[i]
if i%2 == 0 {
if err := trie.Insert(item.key, item.value); err != nil {
panic(err)
}
} else {
if err := trie.Update(item.key, item.value); err != nil {
panic(err)
}
}
}
checkSize(size / 2)
// Check the first half of the dict.
for i := 0; i < size/2; i++ {
exist(i)
}
log.Println("first half OK")
// Delete even items in the first half.
for i := 0; i < size/2; i += 2 {
err := trie.Delete(dict[i].key)
failIfError(err)
}
checkSize(size / 2 / 2)
// Make sure even items were deleted, and the rest are fine.
for i := 0; i < size/2; i++ {
if i%2 == 0 {
notExist(i)
} else {
exist(i)
}
}
log.Println("first half odd OK")
// Insert the second half of the dict.
for i := size / 2; i < size; i++ {
item := dict[i]
trie.Insert(item.key, item.value)
}
checkSize(size/2/2 + (size - size/2))
for i := 0; i < size/2; i++ {
if i%2 == 0 {
notExist(i)
} else {
exist(i)
}
}
log.Println("first half odd still OK")
// Delete even items in the second half.
half := size / 2
if half%2 == 1 {
half++
}
for i := half; i < size; i += 2 {
err := trie.Delete(dict[i].key)
failIfError(err)
}
// Make sure even items were deleted, and the rest are fine.
for i := 0; i < size; i++ {
if i%2 == 0 {
notExist(i)
} else {
exist(i)
}
}
log.Println("odd OK")
// Insert all even terms.
for i := 0; i < size; i += 2 {
item := dict[i]
notExist(i)
trie.Update([]byte(item.key), item.value)
}
for i := 0; i < size; i += 1 {
exist(i)
}
log.Println("all OK")
// Insert every item again, should be fine.
for i := 1; i < size; i++ {
item := dict[i]
trie.Insert([]byte(item.key), item.value)
}
for i := 1; i < size; i += 2 {
exist(i)
}
log.Println("still OK")
}
func failIfError(err error) {
if err != nil {
panic(err)
}
}
// Package cedar-go implements double-array trie.
//
// It is a golang port of cedar (http://www.tkl.iis.u-tokyo.ac.jp/~ynaga/cedar) which is written in C++ by Naoki Yoshinaga.
// Currently cedar-go implements the `reduced` verion of cedar.
// This package is not thread safe if there is one goroutine doing
// insertions or deletions.
//
// Note
//
// key must be `[]byte` without zero items,
// while value must be integer in the range [0, 2<<63-2] or [0, 2<<31-2] depends on the platform.
package cedar
package cedar
import "errors"
var (
ErrInvalidDataType = errors.New("cedar: invalid datatype")
ErrInvalidValue = errors.New("cedar: invalid value")
ErrInvalidKey = errors.New("cedar: invalid key")
ErrNoPath = errors.New("cedar: no path")
ErrNoValue = errors.New("cedar: no value")
)
package cedar_test
import (
"fmt"
"github.com/adamzy/cedar-go"
)
var trie *cedar.Cedar
func Example() {
trie = cedar.New()
// Insert key-value pairs.
// The order of insertion is not important.
trie.Insert([]byte("How many"), 0)
trie.Insert([]byte("How many loved"), 1)
trie.Insert([]byte("How many loved your moments"), 2)
trie.Insert([]byte("How many loved your moments of glad grace"), 3)
trie.Insert([]byte("姑苏"), 4)
trie.Insert([]byte("姑苏城外"), 5)
trie.Insert([]byte("姑苏城外寒山寺"), 6)
// Get the associated value of a key directly.
value, _ := trie.Get([]byte("How many loved your moments of glad grace"))
fmt.Println(value)
// Or, use `jump` to get the id of the trie node fist,
id, _ := trie.Jump([]byte("How many loved your moments"), 0)
// then get the key and the value.
key, _ := trie.Key(id)
value, _ = trie.Value(id)
fmt.Printf("%d\t%s:%v\n", id, key, value)
// Output:
// 3
// 281 How many loved your moments:2
}
func Example_prefixMatch() {
fmt.Println("id\tkey:value")
for _, id := range trie.PrefixMatch([]byte("How many loved your moments of glad grace"), 0) {
key, _ := trie.Key(id)
value, _ := trie.Value(id)
fmt.Printf("%d\t%s:%v\n", id, key, value)
}
// Output:
// id key:value
// 262 How many:0
// 268 How many loved:1
// 281 How many loved your moments:2
// 296 How many loved your moments of glad grace:3
}
func Example_prefixPredict() {
fmt.Println("id\tkey:value")
for _, id := range trie.PrefixPredict([]byte("姑苏"), 0) {
key, _ := trie.Key(id)
value, _ := trie.Value(id)
fmt.Printf("%d\t%s:%v\n", id, key, value)
}
// Output:
// id key:value
// 303 姑苏:4
// 309 姑苏城外:5
// 318 姑苏城外寒山寺:6
}
func Example_saveAndLoad() {
trie.SaveToFile("cedar.gob", "gob")
trie.SaveToFile("cedar.json", "json")
trie.LoadFromFile("cedar.gob", "gob")
trie.LoadFromFile("cedar.json", "json")
}
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment