Simple and Powerful ORM for Go, support mysql,postgres,tidb,sqlite3,mssql,oracle https://xorm.io
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

session_find.go 12 KiB

8 months ago
3 years ago
3 years ago
Fix slice of struct not cache bug (#895) Fix failure caused by nil bean Judge both type of struct and pointer in case of out-of-range Fix issue #894 Add test for join subquery (#1528) Fix test Fix subquery with schema Add test for join subquery Add makefile (#1531) Fix drone Fix ci Add deps Improve drone Fix envs Add makefile Reviewed-on: https://gitea.com/xorm/xorm/pulls/1531 Add password for postgres drone image (#1530) Add password for postgres drone image Reviewed-on: https://gitea.com/xorm/xorm/pulls/1530 format time when sqlTypeName is core.Varchar (#1026) fix time test add test for time format sign codes according to contributing rules. format time when sqlTypeName is core.Varchar. Same with core.DateTime or core.TimeStamp Add test for second insert error (#1527) Add test for second insert error Reviewed-on: https://gitea.com/xorm/xorm/pulls/1527 Add tests for table name (#1517) add tests for table name Fix test (#1526) Fix test Reviewed-on: https://gitea.com/xorm/xorm/pulls/1526 Fix test (#1526) Fix test Reviewed-on: https://gitea.com/xorm/xorm/pulls/1526 Fix wrong warning log on autoincrement column when sync table (#1525) improve doc Fix wrong warning log on autoincrement column when sync table Reviewed-on: https://gitea.com/xorm/xorm/pulls/1525 Fixed Join strings on func Exist (#1520) fix test fixed Join strings on func Exist Co-authored-by: Tomofumi Kusana <tkusana@morisawa.co.jp> Reviewed-on: https://gitea.com/xorm/xorm/pulls/1520 For nullable columns, store nil values as NULL (#531) Merge branch 'master' into jcsalem/fix/nil_ptr_is_nullable fix bug when buffersize with iterate (#941) Merge branch 'master' into lunny/fix_buffer_iterate Exclude schema from index name (#1505) Merge branch 'master' into fix-schema-idx SetExpr support more go types (#1499) Improve tests SetExpr support more go types fix vet fix drone lint remove go1.10 test on drone Reviewed-on: https://gitea.com/xorm/xorm/pulls/1499 fix vet fix drone lint remove go1.10 test on drone Exclude schema from the index name Co-authored-by: Guillermo Prandi <guillep2k@users.noreply.github.com> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Reviewed-on: https://gitea.com/xorm/xorm/pulls/1505 fix test fix bug fix bug when buffersize with iterate SetExpr support more go types (#1499) Improve tests SetExpr support more go types fix vet fix drone lint remove go1.10 test on drone Reviewed-on: https://gitea.com/xorm/xorm/pulls/1499 fix vet fix drone lint remove go1.10 test on drone Fix update with Alias (#1455) Co-authored-by: Guillermo Prandi <guillep2k@noreply.gitea.io> Reviewed-on: https://gitea.com/xorm/xorm/pulls/941 fix update map with version (#1448) fix test fix update map with version SetExpr support more go types (#1499) Improve tests SetExpr support more go types fix vet fix drone lint remove go1.10 test on drone Reviewed-on: https://gitea.com/xorm/xorm/pulls/1499 fix vet fix drone lint remove go1.10 test on drone Fix update with Alias (#1455) Reviewed-on: https://gitea.com/xorm/xorm/pulls/1448 Exclude schema from index name (#1505) Merge branch 'master' into fix-schema-idx SetExpr support more go types (#1499) Improve tests SetExpr support more go types fix vet fix drone lint remove go1.10 test on drone Reviewed-on: https://gitea.com/xorm/xorm/pulls/1499 fix vet fix drone lint remove go1.10 test on drone Exclude schema from the index name Co-authored-by: Guillermo Prandi <guillep2k@users.noreply.github.com> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Reviewed-on: https://gitea.com/xorm/xorm/pulls/1505 SetExpr support more go types (#1499) Improve tests SetExpr support more go types fix vet fix drone lint remove go1.10 test on drone Reviewed-on: https://gitea.com/xorm/xorm/pulls/1499 For nullable columns, store nil values as NULL fix vet fix drone lint remove go1.10 test on drone Fix update with Alias (#1455) Improve ci tests (#1477) Rewrite Engine.QuoteTo() to accept multi-part identifiers (#1476) Support local sql log (#1338) Fix go mod and update version (#1460) Move github.com/go-xorm/xorm to xorm.io/xorm (#1459) add support custom type Nullfloat64 (#1450) fix bug when query map condtion with no quote (#1449) Don't warn when bool column default is 1 but not true (#1447) * don't warn when bool column default is 1 but not true * fix default case sensitive Fix sync2 with custom table name (#1445) * fix sync2 with custom table name * fix bug on postgres * fix bug on postgres fix bug when update with setexpr (#1446) add tidb tests on drone ci (#1444) improve sync2 (#1443) Fix wrong dbmetas (#1442) * add tests for db metas * add more tests * fix bug on mssql Fix default value parse bugs (#1437) * fix default value * fix default value tags * fix postgres default * fix default on postgres * fix default on postgres * fix mssql default fix arg conversion (#1441) * fix arg conversion * fix bugs * fix bug on postgres * use traditional positional parameters on insert into select * remove unnecessary tests upgrade core (#1440) add tests (#1439) ... Co-authored-by: shanyy <shanyy163@163.com> Co-authored-by: kusana <kusana@noreply.gitea.io> Co-authored-by: Guillermo Prandi <guillep2k@noreply.gitea.io> Co-authored-by: Guillermo Prandi <guillep2k@noreply@gitea.io> Co-authored-by: yudppp <yu.d.ppp@gmail.com> Reviewed-on: https://gitea.com/xorm/xorm/pulls/895
1 month ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482
  1. // Copyright 2016 The Xorm Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package xorm
  5. import (
  6. "errors"
  7. "fmt"
  8. "reflect"
  9. "xorm.io/builder"
  10. "xorm.io/xorm/caches"
  11. "xorm.io/xorm/internal/statements"
  12. "xorm.io/xorm/internal/utils"
  13. "xorm.io/xorm/schemas"
  14. )
  15. const (
  16. tpStruct = iota
  17. tpNonStruct
  18. )
  19. // Find retrieve records from table, condiBeans's non-empty fields
  20. // are conditions. beans could be []Struct, []*Struct, map[int64]Struct
  21. // map[int64]*Struct
  22. func (session *Session) Find(rowsSlicePtr interface{}, condiBean ...interface{}) error {
  23. if session.isAutoClose {
  24. defer session.Close()
  25. }
  26. return session.find(rowsSlicePtr, condiBean...)
  27. }
  28. // FindAndCount find the results and also return the counts
  29. func (session *Session) FindAndCount(rowsSlicePtr interface{}, condiBean ...interface{}) (int64, error) {
  30. if session.isAutoClose {
  31. defer session.Close()
  32. }
  33. session.autoResetStatement = false
  34. err := session.find(rowsSlicePtr, condiBean...)
  35. if err != nil {
  36. return 0, err
  37. }
  38. sliceValue := reflect.Indirect(reflect.ValueOf(rowsSlicePtr))
  39. if sliceValue.Kind() != reflect.Slice && sliceValue.Kind() != reflect.Map {
  40. return 0, errors.New("needs a pointer to a slice or a map")
  41. }
  42. sliceElementType := sliceValue.Type().Elem()
  43. if sliceElementType.Kind() == reflect.Ptr {
  44. sliceElementType = sliceElementType.Elem()
  45. }
  46. session.autoResetStatement = true
  47. if session.statement.SelectStr != "" {
  48. session.statement.SelectStr = ""
  49. }
  50. if session.statement.OrderStr != "" {
  51. session.statement.OrderStr = ""
  52. }
  53. // session has stored the conditions so we use `unscoped` to avoid duplicated condition.
  54. return session.Unscoped().Count(reflect.New(sliceElementType).Interface())
  55. }
  56. func (session *Session) find(rowsSlicePtr interface{}, condiBean ...interface{}) error {
  57. defer session.resetStatement()
  58. if session.statement.LastError != nil {
  59. return session.statement.LastError
  60. }
  61. sliceValue := reflect.Indirect(reflect.ValueOf(rowsSlicePtr))
  62. var isSlice = sliceValue.Kind() == reflect.Slice
  63. var isMap = sliceValue.Kind() == reflect.Map
  64. if !isSlice && !isMap {
  65. return errors.New("needs a pointer to a slice or a map")
  66. }
  67. sliceElementType := sliceValue.Type().Elem()
  68. var tp = tpStruct
  69. if session.statement.RefTable == nil {
  70. if sliceElementType.Kind() == reflect.Ptr {
  71. if sliceElementType.Elem().Kind() == reflect.Struct {
  72. pv := reflect.New(sliceElementType.Elem())
  73. if err := session.statement.SetRefValue(pv); err != nil {
  74. return err
  75. }
  76. } else {
  77. tp = tpNonStruct
  78. }
  79. } else if sliceElementType.Kind() == reflect.Struct {
  80. pv := reflect.New(sliceElementType)
  81. if err := session.statement.SetRefValue(pv); err != nil {
  82. return err
  83. }
  84. } else {
  85. tp = tpNonStruct
  86. }
  87. }
  88. var (
  89. table = session.statement.RefTable
  90. addedTableName = (len(session.statement.JoinStr) > 0)
  91. autoCond builder.Cond
  92. )
  93. if tp == tpStruct {
  94. if !session.statement.NoAutoCondition && len(condiBean) > 0 {
  95. var err error
  96. autoCond, err = session.statement.BuildConds(table, condiBean[0], true, true, false, true, addedTableName)
  97. if err != nil {
  98. return err
  99. }
  100. } else {
  101. if col := table.DeletedColumn(); col != nil && !session.statement.GetUnscoped() { // tag "deleted" is enabled
  102. autoCond = session.statement.CondDeleted(col)
  103. }
  104. }
  105. }
  106. // if it's a map with Cols but primary key not in column list, we still need the primary key
  107. if isMap && !session.statement.ColumnMap.IsEmpty() {
  108. for _, k := range session.statement.RefTable.PrimaryKeys {
  109. session.statement.ColumnMap.Add(k)
  110. }
  111. }
  112. sqlStr, args, err := session.statement.GenFindSQL(autoCond)
  113. if err != nil {
  114. return err
  115. }
  116. if session.statement.ColumnMap.IsEmpty() && session.canCache() {
  117. if cacher := session.engine.GetCacher(session.statement.TableName()); cacher != nil &&
  118. !session.statement.IsDistinct &&
  119. !session.statement.GetUnscoped() {
  120. err = session.cacheFind(sliceElementType, sqlStr, rowsSlicePtr, args...)
  121. if err != ErrCacheFailed {
  122. return err
  123. }
  124. err = nil // !nashtsai! reset err to nil for ErrCacheFailed
  125. session.engine.logger.Warnf("Cache Find Failed")
  126. }
  127. }
  128. return session.noCacheFind(table, sliceValue, sqlStr, args...)
  129. }
  130. func (session *Session) noCacheFind(table *schemas.Table, containerValue reflect.Value, sqlStr string, args ...interface{}) error {
  131. rows, err := session.queryRows(sqlStr, args...)
  132. if err != nil {
  133. return err
  134. }
  135. defer rows.Close()
  136. fields, err := rows.Columns()
  137. if err != nil {
  138. return err
  139. }
  140. var newElemFunc func(fields []string) reflect.Value
  141. elemType := containerValue.Type().Elem()
  142. var isPointer bool
  143. if elemType.Kind() == reflect.Ptr {
  144. isPointer = true
  145. elemType = elemType.Elem()
  146. }
  147. if elemType.Kind() == reflect.Ptr {
  148. return errors.New("pointer to pointer is not supported")
  149. }
  150. newElemFunc = func(fields []string) reflect.Value {
  151. switch elemType.Kind() {
  152. case reflect.Slice:
  153. slice := reflect.MakeSlice(elemType, len(fields), len(fields))
  154. x := reflect.New(slice.Type())
  155. x.Elem().Set(slice)
  156. return x
  157. case reflect.Map:
  158. mp := reflect.MakeMap(elemType)
  159. x := reflect.New(mp.Type())
  160. x.Elem().Set(mp)
  161. return x
  162. }
  163. return reflect.New(elemType)
  164. }
  165. var containerValueSetFunc func(*reflect.Value, schemas.PK) error
  166. if containerValue.Kind() == reflect.Slice {
  167. containerValueSetFunc = func(newValue *reflect.Value, pk schemas.PK) error {
  168. if isPointer {
  169. containerValue.Set(reflect.Append(containerValue, newValue.Elem().Addr()))
  170. } else {
  171. containerValue.Set(reflect.Append(containerValue, newValue.Elem()))
  172. }
  173. return nil
  174. }
  175. } else {
  176. keyType := containerValue.Type().Key()
  177. if len(table.PrimaryKeys) == 0 {
  178. return errors.New("don't support multiple primary key's map has non-slice key type")
  179. }
  180. if len(table.PrimaryKeys) > 1 && keyType.Kind() != reflect.Slice {
  181. return errors.New("don't support multiple primary key's map has non-slice key type")
  182. }
  183. containerValueSetFunc = func(newValue *reflect.Value, pk schemas.PK) error {
  184. keyValue := reflect.New(keyType)
  185. err := convertPKToValue(table, keyValue.Interface(), pk)
  186. if err != nil {
  187. return err
  188. }
  189. if isPointer {
  190. containerValue.SetMapIndex(keyValue.Elem(), newValue.Elem().Addr())
  191. } else {
  192. containerValue.SetMapIndex(keyValue.Elem(), newValue.Elem())
  193. }
  194. return nil
  195. }
  196. }
  197. if elemType.Kind() == reflect.Struct {
  198. var newValue = newElemFunc(fields)
  199. dataStruct := utils.ReflectValue(newValue.Interface())
  200. tb, err := session.engine.tagParser.ParseWithCache(dataStruct)
  201. if err != nil {
  202. return err
  203. }
  204. err = session.rows2Beans(rows, fields, tb, newElemFunc, containerValueSetFunc)
  205. rows.Close()
  206. if err != nil {
  207. return err
  208. }
  209. return session.executeProcessors()
  210. }
  211. for rows.Next() {
  212. var newValue = newElemFunc(fields)
  213. bean := newValue.Interface()
  214. switch elemType.Kind() {
  215. case reflect.Slice:
  216. err = rows.ScanSlice(bean)
  217. case reflect.Map:
  218. err = rows.ScanMap(bean)
  219. default:
  220. err = rows.Scan(bean)
  221. }
  222. if err != nil {
  223. return err
  224. }
  225. if err := containerValueSetFunc(&newValue, nil); err != nil {
  226. return err
  227. }
  228. }
  229. return nil
  230. }
  231. func convertPKToValue(table *schemas.Table, dst interface{}, pk schemas.PK) error {
  232. cols := table.PKColumns()
  233. if len(cols) == 1 {
  234. return convertAssign(dst, pk[0])
  235. }
  236. dst = pk
  237. return nil
  238. }
  239. func (session *Session) cacheFind(t reflect.Type, sqlStr string, rowsSlicePtr interface{}, args ...interface{}) (err error) {
  240. if !session.canCache() ||
  241. utils.IndexNoCase(sqlStr, "having") != -1 ||
  242. utils.IndexNoCase(sqlStr, "group by") != -1 {
  243. return ErrCacheFailed
  244. }
  245. tableName := session.statement.TableName()
  246. cacher := session.engine.cacherMgr.GetCacher(tableName)
  247. if cacher == nil {
  248. return nil
  249. }
  250. for _, filter := range session.engine.dialect.Filters() {
  251. sqlStr = filter.Do(sqlStr)
  252. }
  253. newsql := session.statement.ConvertIDSQL(sqlStr)
  254. if newsql == "" {
  255. return ErrCacheFailed
  256. }
  257. table := session.statement.RefTable
  258. ids, err := caches.GetCacheSql(cacher, tableName, newsql, args)
  259. if err != nil {
  260. rows, err := session.queryRows(newsql, args...)
  261. if err != nil {
  262. return err
  263. }
  264. defer rows.Close()
  265. var i int
  266. ids = make([]schemas.PK, 0)
  267. for rows.Next() {
  268. i++
  269. if i > 500 {
  270. session.engine.logger.Debugf("[cacheFind] ids length > 500, no cache")
  271. return ErrCacheFailed
  272. }
  273. var res = make([]string, len(table.PrimaryKeys))
  274. err = rows.ScanSlice(&res)
  275. if err != nil {
  276. return err
  277. }
  278. var pk schemas.PK = make([]interface{}, len(table.PrimaryKeys))
  279. for i, col := range table.PKColumns() {
  280. pk[i], err = session.engine.idTypeAssertion(col, res[i])
  281. if err != nil {
  282. return err
  283. }
  284. }
  285. ids = append(ids, pk)
  286. }
  287. session.engine.logger.Debugf("[cache] cache sql: %v, %v, %v, %v, %v", ids, tableName, sqlStr, newsql, args)
  288. err = caches.PutCacheSql(cacher, ids, tableName, newsql, args)
  289. if err != nil {
  290. return err
  291. }
  292. } else {
  293. session.engine.logger.Debugf("[cache] cache hit sql: %v, %v, %v, %v", tableName, sqlStr, newsql, args)
  294. }
  295. sliceValue := reflect.Indirect(reflect.ValueOf(rowsSlicePtr))
  296. ididxes := make(map[string]int)
  297. var ides []schemas.PK
  298. var temps = make([]interface{}, len(ids))
  299. for idx, id := range ids {
  300. sid, err := id.ToString()
  301. if err != nil {
  302. return err
  303. }
  304. bean := cacher.GetBean(tableName, sid)
  305. // fix issue #894
  306. isHit := func() (ht bool) {
  307. if bean == nil {
  308. ht = false
  309. return
  310. }
  311. ckb := reflect.ValueOf(bean).Elem().Type()
  312. ht = ckb == t
  313. if !ht && t.Kind() == reflect.Ptr {
  314. ht = t.Elem() == ckb
  315. }
  316. return
  317. }
  318. if !isHit() {
  319. ides = append(ides, id)
  320. ididxes[sid] = idx
  321. } else {
  322. session.engine.logger.Debugf("[cache] cache hit bean: %v, %v, %v", tableName, id, bean)
  323. pk, err := session.engine.IDOf(bean)
  324. if err != nil {
  325. return err
  326. }
  327. xid, err := pk.ToString()
  328. if err != nil {
  329. return err
  330. }
  331. if sid != xid {
  332. session.engine.logger.Errorf("[cache] error cache: %v, %v, %v", xid, sid, bean)
  333. return ErrCacheFailed
  334. }
  335. temps[idx] = bean
  336. }
  337. }
  338. if len(ides) > 0 {
  339. slices := reflect.New(reflect.SliceOf(t))
  340. beans := slices.Interface()
  341. statement := session.statement
  342. session.statement = statements.NewStatement(
  343. session.engine.dialect,
  344. session.engine.tagParser,
  345. session.engine.DatabaseTZ,
  346. )
  347. if len(table.PrimaryKeys) == 1 {
  348. ff := make([]interface{}, 0, len(ides))
  349. for _, ie := range ides {
  350. ff = append(ff, ie[0])
  351. }
  352. session.In("`"+table.PrimaryKeys[0]+"`", ff...)
  353. } else {
  354. for _, ie := range ides {
  355. cond := builder.NewCond()
  356. for i, name := range table.PrimaryKeys {
  357. cond = cond.And(builder.Eq{"`" + name + "`": ie[i]})
  358. }
  359. session.Or(cond)
  360. }
  361. }
  362. err = session.NoCache().Table(tableName).find(beans)
  363. if err != nil {
  364. return err
  365. }
  366. session.statement = statement
  367. vs := reflect.Indirect(reflect.ValueOf(beans))
  368. for i := 0; i < vs.Len(); i++ {
  369. rv := vs.Index(i)
  370. if rv.Kind() != reflect.Ptr {
  371. rv = rv.Addr()
  372. }
  373. id, err := session.engine.idOfV(rv)
  374. if err != nil {
  375. return err
  376. }
  377. sid, err := id.ToString()
  378. if err != nil {
  379. return err
  380. }
  381. bean := rv.Interface()
  382. temps[ididxes[sid]] = bean
  383. session.engine.logger.Debugf("[cache] cache bean: %v, %v, %v, %v", tableName, id, bean, temps)
  384. cacher.PutBean(tableName, sid, bean)
  385. }
  386. }
  387. for j := 0; j < len(temps); j++ {
  388. bean := temps[j]
  389. if bean == nil {
  390. session.engine.logger.Warnf("[cache] cache no hit: %v, %v, %v", tableName, ids[j], temps)
  391. // return errors.New("cache error") // !nashtsai! no need to return error, but continue instead
  392. continue
  393. }
  394. if sliceValue.Kind() == reflect.Slice {
  395. if t.Kind() == reflect.Ptr {
  396. sliceValue.Set(reflect.Append(sliceValue, reflect.ValueOf(bean)))
  397. } else {
  398. sliceValue.Set(reflect.Append(sliceValue, reflect.Indirect(reflect.ValueOf(bean))))
  399. }
  400. } else if sliceValue.Kind() == reflect.Map {
  401. var key = ids[j]
  402. keyType := sliceValue.Type().Key()
  403. var ikey interface{}
  404. if len(key) == 1 {
  405. ikey, err = str2PK(fmt.Sprintf("%v", key[0]), keyType)
  406. if err != nil {
  407. return err
  408. }
  409. } else {
  410. if keyType.Kind() != reflect.Slice {
  411. return errors.New("table have multiple primary keys, key is not schemas.PK or slice")
  412. }
  413. ikey = key
  414. }
  415. if t.Kind() == reflect.Ptr {
  416. sliceValue.SetMapIndex(reflect.ValueOf(ikey), reflect.ValueOf(bean))
  417. } else {
  418. sliceValue.SetMapIndex(reflect.ValueOf(ikey), reflect.Indirect(reflect.ValueOf(bean)))
  419. }
  420. }
  421. }
  422. return nil
  423. }