package integration

import (
	"bytes"
	"context"
	"fmt"
	"mime/multipart"
	"net/http"
	"net/url"
	"strings"
	"testing"
	"time"

	"github.com/contiamo/dev/tests/integration/suite"
	uuid "github.com/satori/go.uuid"
	"github.com/stretchr/testify/require"
)

type Payload suite.Payload

func TestDatasourceFlows(t *testing.T) {
	var (
		externalDS   = "foodmart"
		externalDSID string

		managedDS      = "pity"
		managedTable   = "the_foo"
		managedDSID    string
		fileID         string
		managedTableID string

		virtualDSID string
		virtualDS   = "virtual_food"
		viewName    = "view"
		viewTwoName = "view2"
	)

	s := suite.New(t)

	suffix := time.Now().Format("2006-01-02T15:04:05Z07:00")
	err := s.Setup("e2e-" + suffix)
	require.NoError(t, err)

	// log the project so that we can manually find it in the logs incase we need to manually
	// run cleanup
	fmt.Printf("test project /auth/api/v2/tenants/%s/realms/%s\n", s.TenantID, s.ProjectID)

	// urls that we will send DELETE to
	cleanup := []string{
		fmt.Sprintf("/auth/api/v2/tenants/%s/realms/%s", s.TenantID, s.ProjectID),
	}

	t.Cleanup(func() {
		if s.Config.SkipCleanup {
			fmt.Println("skipping test cleanup")
			return
		}

		for _, path := range cleanup {
			// ignore any potential errors, on a successful test they will be 404
			_ = s.Delete(path)
		}
	})

	s.Run("add second user to project", func(s *suite.Suite) {
		s.Run("get user id", func(s *suite.Suite) {
			path := fmt.Sprintf("/auth/api/v2/tenants/%s/users?page=1&search=%s", s.TenantID, url.QueryEscape(s.Config.UserEmail))
			resp := s.Get(path)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusOK, resp.StatusCode, "incorrect status code when searching for second user")

			data := s.ParsePayload(resp.Body)
			idVal, _ := s.JqVal(data, ".data[0].id").(string)
			s.Require().NotEmpty(idVal, "search response: %#v", data)

			s.UserIDs[s.Config.UserEmail] = idVal
		})

		path := fmt.Sprintf("auth/api/v2/tenants/%s/realms/%s/members", s.TenantID, s.ProjectID)
		payload := Payload{
			"userId": s.UserIDs[s.Config.UserEmail],
		}
		resp := s.Post(path, payload)
		defer resp.Body.Close()
		s.Require().Equal(http.StatusOK, resp.StatusCode, "incorrect status code when adding the second user")

	})

	s.Run("external datasource", func(s *suite.Suite) {

		s.Run("create new external datasource", func(s *suite.Suite) {
			path := fmt.Sprintf("/hub/api/v1/%s/datasources", s.ProjectID)
			payload := Payload{
				"name":           externalDS,
				"type":           "external",
				"technology":     "hsqldb-foodmart",
				"isPublic":       true,
				"connectionInfo": Payload{},
			}
			resp := s.Post(path, payload)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusCreated, resp.StatusCode)
			createResponse := s.ParsePayload(resp.Body)
			id, ok := createResponse["id"].(string)
			s.Require().True(ok)
			externalDSID = id

			cleanup = append(cleanup, fmt.Sprintf("/hub/api/v1/%s/datasources/%s", s.ProjectID, externalDSID))

			// we need to wait some time to ensure that metadata is synced
			time.Sleep(11 * time.Second)
			s.Run("check hub effects", func(s *suite.Suite) {
				s.Run("check if we can get it by id", func(s *suite.Suite) {
					path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s", s.ProjectID, externalDSID)
					resp := s.Get(path)
					defer resp.Body.Close()
					s.Require().Equal(http.StatusOK, resp.StatusCode)
					data := s.ParsePayload(resp.Body)
					hits := s.Jq(data, `.data.name`)
					s.Require().Len(hits, 1)
					s.Require().Equal(externalDS, hits[0])

					s.Run("check that a sync task was scheduled by the time we GET it now", func(s *suite.Suite) {
						taskIdHits := s.Jq(data, `.data.syncTask.taskId`)
						s.Require().NotEmpty(taskIdHits[0])
					})
				})

				s.Run("check if we can get it by name", func(s *suite.Suite) {
					path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s", s.ProjectID, externalDS)
					resp := s.Get(path)
					defer resp.Body.Close()
					s.Require().Equal(http.StatusOK, resp.StatusCode)
					data := s.ParsePayload(resp.Body)
					hits := s.Jq(data, `.data.id`)
					s.Require().Len(hits, 1)
					s.Require().Equal(externalDSID, hits[0])
				})

				s.Run("check if we find it in the datasource list", func(s *suite.Suite) {
					path := fmt.Sprintf("/hub/api/v1/%s/resources/search?kinds=datasource&query=%s", s.ProjectID, externalDS)
					resp := s.Get(path)
					defer resp.Body.Close()
					s.Require().Equal(http.StatusOK, resp.StatusCode)
					data := s.ParsePayload(resp.Body)
					hits := s.Jq(data, `.data[] | select(.resource.name == "%s")`, externalDS)
					s.Require().Len(hits, 1)
				})

			})

			s.Run("check pantheon effects", func(s *suite.Suite) {
				s.Run("check if we can query the datasource", func(s *suite.Suite) {
					path := fmt.Sprintf("/hub/api/v1/%s/query", s.ProjectID)
					resp := s.Post(path, Payload{
						"sql": fmt.Sprintf(`SELECT account_id FROM "%s"."account"`, externalDS),
					})
					defer resp.Body.Close()
					data := s.ParsePayload(resp.Body)
					hits := s.Jq(data, `.rows[]`)
					s.Require().Len(hits, 11)
				})

				s.Run("check if we can profile the query", func(s *suite.Suite) {
					path := fmt.Sprintf("/hub/api/v1/%s/profile/sql", s.ProjectID)
					resp := s.Post(path, Payload{
						"projectId": s.ProjectID,
						"sql":       fmt.Sprintf(`SELECT account_id FROM "%s"."account"`, externalDS),
					})
					defer resp.Body.Close()
					data := s.ParsePayload(resp.Body)
					columns, ok := data["columns"].(map[string]interface{})
					s.Require().True(ok, "unknown response %+v", data)
					s.Require().Len(columns, 1, "unknown response %+v", data)

					accountIDProfile, ok := columns["account_id"].(map[string]interface{})
					s.Require().True(ok, "unknown response %+v", columns)
					s.Require().Equal(accountIDProfile["dataType"], "integer", "unknown response %+v", accountIDProfile)
				})

				s.Run("check if we can query in pgql", func(s *suite.Suite) {
					path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s/graphql", s.ProjectID, "foodmart")
					resp := s.Post(path, Payload{
						"query":     `{account(limit:1, where:{account_id: {_eq: 1000}}) {account_id}}`,
						"variables": nil,
					})

					defer resp.Body.Close()
					data := s.ParsePayload(resp.Body)

					// stupid ECMAscript ints!
					id := s.JqVal(data, `.data.account[0].account_id`).(float64)
					s.Require().Equal(float64(1000), id)
				})
			})
		})

		s.Run("reject to delete the table", func(s *suite.Suite) {
			path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s/tables/account", s.ProjectID, externalDSID)
			resp := s.Delete(path)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusUnprocessableEntity, resp.StatusCode)

			s.Run("check if we can still query the datasource", func(s *suite.Suite) {
				path := fmt.Sprintf("/hub/api/v1/%s/query", s.ProjectID)
				resp := s.Post(path, Payload{
					"sql": fmt.Sprintf(`SELECT account_id FROM "%s"."account"`, externalDS),
				})
				defer resp.Body.Close()
				data := s.ParsePayload(resp.Body)
				hits := s.Jq(data, `.rows[]`)
				s.Require().Len(hits, 11)
			})
		})

	})

	s.Run("permissions", func(s *suite.Suite) {
		var requestID string
		s.Run("request access", func(s *suite.Suite) {
			s.SetCurrentUser(s.Config.UserEmail, s.Config.UserPassword)

			path := fmt.Sprintf("/hub/api/v1/%s/resources/%s/requests", s.ProjectID, externalDSID)
			payload := Payload{"comment": "please"}
			resp := s.Post(path, payload)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusCreated, resp.StatusCode)

			data := s.ParsePayload(resp.Body)
			id, ok := data["id"].(string)
			s.Require().True(ok, "can not get request id")
			requestID = id
		})

		s.Run("resolve access request", func(s *suite.Suite) {
			s.SetCurrentUser(s.Config.AdminEmail, s.Config.AdminPassword)

			path := fmt.Sprintf("/hub/api/v1/%s/resources/%s/requests/%s", s.ProjectID, externalDSID, requestID)
			payload := Payload{
				"comment": "ok",
				"role":    "query",
				"state":   "granted",
			}
			resp := s.Put(path, payload)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusOK, resp.StatusCode)
		})

		s.Run("check query role", func(s *suite.Suite) {
			s.SetCurrentUser(s.Config.UserEmail, s.Config.UserPassword)

			path := fmt.Sprintf("/hub/api/v1/%s/query", s.ProjectID)
			resp := s.Post(path, Payload{
				"sql": fmt.Sprintf(`SELECT account_id FROM "%s"."account"`, externalDS),
			})
			defer resp.Body.Close()
			data := s.ParsePayload(resp.Body)
			hits := s.Jq(data, `.rows[]`)
			s.Require().Len(hits, 11)
		})

		s.Run("elevate permissions", func(s *suite.Suite) {
			s.SetCurrentUser(s.Config.AdminEmail, s.Config.AdminPassword)

			path := fmt.Sprintf("hub/api/v1/%s/resources/%s/permissions", s.ProjectID, externalDSID)
			payload := Payload{
				"principalId": s.UserIDs[s.Config.UserEmail],
				"role":        "steward",
			}
			resp := s.Post(path, payload)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusCreated, resp.StatusCode)
		})
	})

	s.SetCurrentUser(s.Config.AdminEmail, s.Config.AdminPassword)

	s.Run("managed datasource", func(s *suite.Suite) {
		s.Run("create new managed datasource", func(s *suite.Suite) {
			path := fmt.Sprintf("/hub/api/v1/%s/datasources", s.ProjectID)
			payload := Payload{
				"name":     managedDS,
				"type":     "managed",
				"isPublic": true,
			}
			resp := s.Post(path, payload)
			defer resp.Body.Close()

			data := s.ParsePayload(resp.Body)
			s.Require().Equal(http.StatusCreated, resp.StatusCode, "payload: %+v", data)
			idVal := s.JqVal(data, ".id")
			s.Require().NotEmpty(idVal, "id is not found in the response")
			var ok bool
			managedDSID, ok = idVal.(string)
			s.Require().True(ok, "ID must be a string")

			cleanup = append(cleanup, fmt.Sprintf("/hub/api/v1/%s/datasources/%s", s.ProjectID, managedDSID))

			s.Run("check if we can get it by id", func(s *suite.Suite) {
				path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s", s.ProjectID, managedDSID)
				resp := s.Get(path)
				defer resp.Body.Close()
				s.Require().Equal(http.StatusOK, resp.StatusCode)
				data := s.ParsePayload(resp.Body)
				s.Require().Equal(managedDS, s.JqVal(data, `.data.name`))
			})
		})

		s.Run("upload csv file", func(s *suite.Suite) {
			path := fmt.Sprintf("/hub/api/v1/%v/datasources/%v/uploads", s.ProjectID, managedDSID)

			body := &bytes.Buffer{}
			writer := multipart.NewWriter(body)
			part, err := writer.CreateFormFile("file", "file.csv")
			s.Require().NoError(err)
			part.Write([]byte("foo,bar,baz\na,2,3\nb,2,3\nc,2,3\n"))
			writer.Close()

			request, err := http.NewRequest(http.MethodPost, s.PlatformURL(path), body)
			s.Require().NoError(err)
			request.Header.Add("Content-Type", writer.FormDataContentType())
			resp, err := s.Do(request)
			s.Require().NoError(err)
			defer resp.Body.Close()

			payload := s.ParsePayload(resp.Body)
			s.Require().Equal(http.StatusOK, resp.StatusCode, "payload: %+v", payload)
			var ok bool
			fileID, ok = payload["fileId"].(string)
			s.Require().True(ok, "payload: %+v", payload)
		})

		s.Run("create table", func(s *suite.Suite) {
			path := fmt.Sprintf("/hub/api/v1/%v/datasources/%v/managed-tables", s.ProjectID, managedDSID)
			resp := s.Post(path, Payload{
				"name":                managedTable,
				"defaultRowErrorMode": "drop",
				"defaultInsertMode":   "upsert",
				"schema": Payload{
					"columns": []Payload{
						{"name": "foo", "type": "text", "required": false},
						{"name": "bar", "type": "integer", "required": false},
						{"name": "baz", "type": "integer", "required": false},
					},
				},
			})
			payload := s.ParsePayload(resp.Body)
			s.Require().Equal(http.StatusCreated, resp.StatusCode, "payload: %+v", payload)
			idVal := s.JqVal(payload, `.data.id`)

			var ok bool
			managedTableID, ok = idVal.(string)
			s.Require().True(ok, "tableIDVal: %+v", idVal)
			s.Require().NotEmpty(managedTableID, "tableID: %+v", managedTableID)
		})

		s.Run("start ingestion", func(s *suite.Suite) {
			path := fmt.Sprintf("/hub/api/v1/%v/datasources/%v/tasks", s.ProjectID, managedDSID)
			resp := s.Post(path, Payload{
				"tableId": managedTableID,
				"fileId":  fileID,
				"schema": Payload{
					"columns": []Payload{
						{"name": "foo", "type": "text", "required": false},
						{"name": "bar", "type": "integer", "required": false},
						{"name": "baz", "type": "integer", "required": false},
					},
				},
				"options": Payload{
					"preferredDateFormat": "monthFirst",
					"delimiter":           ",",
					"decimalSeparator":    "numericComma",
					"encoding":            "ISO-8859-9",
					"commentCharacter":    "#",
					"hasHeader":           true,
					"ignoreHeader":        false,
					"trimSpaces":          false,
				},
				"insertMode": "upsert",
			})
			payload := s.ParsePayload(resp.Body)
			s.Require().Equal(http.StatusCreated, resp.StatusCode, "payload: %+v", payload)
			time.Sleep(3 * time.Second)
		})

		s.Run("side effects", func(s *suite.Suite) {
			s.Run("check if we can query the datasource", func(s *suite.Suite) {
				attempts := 15
				for i := 0; i < attempts; i++ {
					path := fmt.Sprintf("/hub/api/v1/%s/query", s.ProjectID)
					resp := s.Post(path, Payload{
						"sql": fmt.Sprintf(`SELECT * FROM "%s"."%s"`, managedDS, managedTable),
					})
					defer resp.Body.Close()
					data := s.ParsePayload(resp.Body)
					hits := s.Jq(data, `.rows[]`)
					if len(hits) == 3 {
						return
					}
					time.Sleep(1 * time.Second)
				}
				s.Require().True(false, "failed to query data after %d retries", attempts)
			})

			s.Run("check if we can profile the query", func(s *suite.Suite) {
				path := fmt.Sprintf("/hub/api/v1/%s/profile/sql", s.ProjectID)
				resp := s.Post(path, Payload{
					"projectId": s.ProjectID,
					"sql":       fmt.Sprintf(`SELECT foo, bar, baz FROM "%s"."%s"`, managedDS, managedTable),
				})
				defer resp.Body.Close()
				data := s.ParsePayload(resp.Body)
				columns, ok := data["columns"].(map[string]interface{})
				s.Require().True(ok, "unknown response %+v", data)
				s.Require().Len(columns, 3, "unknown response %+v", data)

				profile, ok := columns["foo"].(map[string]interface{})
				s.Require().True(ok, "unknown response %+v", columns)
				s.Require().Equal(profile["dataType"], "string", "unknown response %+v", profile)

				profile, ok = columns["bar"].(map[string]interface{})
				s.Require().True(ok, "unknown response %+v", columns)
				s.Require().Equal(profile["dataType"], "integer", "unknown response %+v", profile)
			})
		})
	})

	s.Run("virtual datasource", func(s *suite.Suite) {
		s.Run("create a virtual datasource through hub", func(s *suite.Suite) {
			path := fmt.Sprintf("/hub/api/v1/%s/datasources", s.ProjectID)
			payload := Payload{
				"name":     virtualDS,
				"type":     "virtual",
				"isPublic": true,
			}
			resp := s.Post(path, payload)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusCreated, resp.StatusCode)
			createResponse := s.ParsePayload(resp.Body)
			id, ok := createResponse["id"].(string)
			s.Require().True(ok)
			virtualDSID = id

			cleanup = append(cleanup, fmt.Sprintf("/hub/api/v1/%s/datasources/%s", s.ProjectID, virtualDSID))
		})

		s.Run("check hub effects", func(s *suite.Suite) {
			s.Run("check if we can get it by id", func(s *suite.Suite) {
				path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s", s.ProjectID, virtualDSID)
				resp := s.Get(path)
				defer resp.Body.Close()
				s.Require().Equal(http.StatusOK, resp.StatusCode)
				data := s.ParsePayload(resp.Body)
				hits := s.Jq(data, `.data.name`)
				s.Require().Len(hits, 1)
				s.Require().Equal(virtualDS, hits[0])
			})

			s.Run("check if we can get it by name", func(s *suite.Suite) {
				path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s", s.ProjectID, virtualDS)
				resp := s.Get(path)
				defer resp.Body.Close()
				s.Require().Equal(http.StatusOK, resp.StatusCode)
				data := s.ParsePayload(resp.Body)
				hits := s.Jq(data, `.data.id`)
				s.Require().Len(hits, 1)
				s.Require().Equal(virtualDSID, hits[0])
			})

			s.Run("check if we find it in the datasource list", func(s *suite.Suite) {
				path := fmt.Sprintf("/hub/api/v1/%s/resources/search?kinds=datasource&query=%s", s.ProjectID, virtualDS)
				resp := s.Get(path)
				defer resp.Body.Close()
				s.Require().Equal(http.StatusOK, resp.StatusCode)
				data := s.ParsePayload(resp.Body)
				hits := s.Jq(data, `.data[] | select(.resource.name == "%s")`, virtualDS)
				s.Require().Len(hits, 1)
			})
		})

		var viewTwoID string
		s.Run("test foodmart view creation", func(s *suite.Suite) {
			s.Run("setup two views", func(s *suite.Suite) {

				path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s/views", s.ProjectID, virtualDSID)
				resp := s.Post(path, Payload{
					"name": viewName,
					"sql":  fmt.Sprintf("SELECT * FROM %s.account", externalDS),
				})
				defer resp.Body.Close()
				s.Require().Equal(http.StatusCreated, resp.StatusCode, "path %s", path)

				resp = s.Post(path, Payload{
					"name": viewTwoName,
					"sql":  fmt.Sprintf("SELECT * FROM %s.account", externalDS),
				})
				defer resp.Body.Close()
				s.Require().Equal(http.StatusCreated, resp.StatusCode, "path %s", path)

				data := s.ParsePayload(resp.Body)
				viewTwoID = s.JqVal(data, ".data.id").(string)
			})

			s.Run("check if we can query the view", func(s *suite.Suite) {
				path := fmt.Sprintf("/hub/api/v1/%s/query", s.ProjectID)
				resp := s.Post(path, Payload{
					"sql": fmt.Sprintf(`SELECT * FROM %s.%s`, virtualDS, viewName),
				})
				defer resp.Body.Close()
				data := s.ParsePayload(resp.Body)
				s.Require().Len(s.Jq(data, `.rows[]`), 11, data)
			})

			s.Run("update view definition", func(s *suite.Suite) {
				path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s/views/%s", s.ProjectID, virtualDSID, viewName)
				resp := s.Put(path, Payload{
					"sql": fmt.Sprintf("SELECT * FROM %s.category", externalDS),
				})
				defer resp.Body.Close()
				s.Require().Equal(http.StatusOK, resp.StatusCode, "path %s", path)
			})

			s.Run("check query effects", func(s *suite.Suite) {
				s.Run("check if we can query the updated view", func(s *suite.Suite) {
					path := fmt.Sprintf("/hub/api/v1/%s/query", s.ProjectID)
					resp := s.Post(path, Payload{
						"sql": fmt.Sprintf(`SELECT * FROM %s.%s`, virtualDS, viewName),
					})
					defer resp.Body.Close()
					data := s.ParsePayload(resp.Body)
					s.Require().Len(s.Jq(data, `.rows[]`), 4, data)
				})

				s.Run("check if we can profile the query", func(s *suite.Suite) {
					path := fmt.Sprintf("/hub/api/v1/%s/profile/sql", s.ProjectID)
					resp := s.Post(path, Payload{
						"projectId": s.ProjectID,
						"sql":       fmt.Sprintf(`SELECT category_id FROM %s.%s`, virtualDS, viewName),
					})
					defer resp.Body.Close()
					data := s.ParsePayload(resp.Body)
					columns, ok := data["columns"].(map[string]interface{})
					s.Require().True(ok, "unknown response %+v", data)
					s.Require().Len(columns, 1, "unknown response %+v", data)

					profile, ok := columns["category_id"].(map[string]interface{})
					s.Require().True(ok, "unknown response %+v", columns)
					s.Require().Equal(profile["dataType"], "string", "unknown response %+v", profile)
				})
			})
		})

		s.Run("test materialization creation", func(s *suite.Suite) {
			s.Run("convert from existing view", func(s *suite.Suite) {
				path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s/views/%s", s.ProjectID, virtualDSID, viewTwoID)
				resp := s.Put(path, Payload{
					"sql": fmt.Sprintf("SELECT * FROM %s.account", externalDS),
					"materialization": Payload{
						"enabled":      true,
						"expiresUnit":  "days",
						"cron":         "0 0 * * *",
						"expiresValue": 1,
					},
				})
				defer resp.Body.Close()
				s.Require().Equal(http.StatusOK, resp.StatusCode, "path %s", path)

				s.Run("check that materialization succeeds", func(s *suite.Suite) {
					ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
					defer cancel()

					var finishedAt interface{}
					var status interface{}
					time.Sleep(2 * time.Second)
					for ctx.Err() == nil {
						path = fmt.Sprintf("/hub/api/v1/%s/datasources/%s/views/%s/tasks", s.ProjectID, virtualDSID, viewTwoID)
						resp = s.Get(path)
						defer resp.Body.Close()

						data := s.ParsePayload(resp.Body)
						s.Require().Equal(http.StatusOK, resp.StatusCode, "path %s: %+v", path, data)

						finishedAt = s.JqVal(data, ".data[0].finishedAt")
						if finishedAt != nil {
							status = s.JqVal(data, ".data[0].status")
							break
						}

						time.Sleep(2 * time.Second)
					}

					s.Require().NotNil(finishedAt, "materialization task did not finish in time")
					s.Require().Equal("finished", status, "materialization did not complete correctly")
				})

				// TODO wait and then query via pantheon, check that it used the physical materialization
			})

			s.Run("check pantheon queries the physical materialization", func(s *suite.Suite) {
				ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
				defer cancel()

				// need to wait for pantheon materialization cache to update
				time.Sleep(5 * time.Second)
				for ctx.Err() == nil {
					queryID := uuid.NewV4()

					path := fmt.Sprintf("/hub/api/v1/%s/query", s.ProjectID)
					resp := s.Post(path, Payload{
						"queryId": queryID.String(),
						"sql":     fmt.Sprintf(`SELECT * FROM %s.%s WHERE account_id = 1000 `, virtualDS, viewTwoName),
					})
					defer resp.Body.Close()
					data := s.ParsePayload(resp.Body)
					s.Require().Len(s.Jq(data, `.rows[]`), 1, data)

					path = fmt.Sprintf("/hub/api/v1/%s/queryHistory/%s", s.ProjectID, queryID.String())
					resp = s.Get(path)
					defer resp.Body.Close()
					data = s.ParsePayload(resp.Body)

					plan := s.JqVal(data, `.backendLogicalPlan`).(string)
					if strings.Contains(plan, "_materialization") {
						return
					}
					// we might still need to wait for pantheon materialization cache to update
					time.Sleep(5 * time.Second)
				}

				s.Require().True(false, "materialization query did go to physical materialization in the expected time")
			})

			s.Run("convert back to view", func(s *suite.Suite) {
				path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s/views/%s", s.ProjectID, virtualDSID, viewTwoName)
				resp := s.Put(path, Payload{
					"sql": fmt.Sprintf("SELECT * FROM %s.account", externalDS),
					"materialization": Payload{
						"enabled": false,
					},
				})
				defer resp.Body.Close()
				s.Require().Equal(http.StatusOK, resp.StatusCode, "path %s", path)
			})

			s.Run("check pantheon queries the virtual view", func(s *suite.Suite) {
				ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
				defer cancel()

				// need to wait for pantheon materialization cache to update
				time.Sleep(2 * time.Second)
				for ctx.Err() == nil {
					queryID := uuid.NewV4()

					path := fmt.Sprintf("/hub/api/v1/%s/query", s.ProjectID)
					resp := s.Post(path, Payload{
						"queryId": queryID.String(),
						"sql":     fmt.Sprintf(`SELECT * FROM %s.%s WHERE account_id = 1000 `, virtualDS, viewTwoName),
					})
					defer resp.Body.Close()
					data := s.ParsePayload(resp.Body)
					s.Require().Len(s.Jq(data, `.rows[]`), 1, data)

					path = fmt.Sprintf("/hub/api/v1/%s/queryHistory/%s", s.ProjectID, queryID.String())
					resp = s.Get(path)
					defer resp.Body.Close()
					data = s.ParsePayload(resp.Body)

					plan := s.JqVal(data, `.backendLogicalPlan`).(string)
					// check that it doesn't contain materialization!
					if !strings.Contains(plan, "_materialization") {
						return
					}
					// we might still need to wait for pantheon materialization cache to update
					time.Sleep(5 * time.Second)
				}

				s.Require().True(false, "query still went to the physical materialization")
			})
		})

	})

	s.Run("catalog search", func(s *suite.Suite) {
		s.Run("create custom fields", func(s *suite.Suite) {
			path := fmt.Sprintf("/hub/api/v2/%s/fields", s.ProjectID)
			body := Payload{
				"kind":        "custom",
				"name":        "custom1",
				"type":        "text",
				"description": "One Love, One Heart, Let's get together and feel alright",
				"filterable":  true,
				"displayName": "Custom One",
			}
			resp := s.Post(path, body)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusCreated, resp.StatusCode, "path %s", path)

			body = Payload{
				"kind":        "custom",
				"name":        "custom2",
				"type":        "boolean",
				"description": "Just the two of us, We can make it if we try, Just the two of us, (Just the two of us), Just the two of us, Building castles in the sky, Just the two of us, You and I",
				"filterable":  true,
				"displayName": "Custom Two",
			}
			s.Post(path, body)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusCreated, resp.StatusCode, "path %s", path)
		})

		s.Run("assign custom fields", func(s *suite.Suite) {
			path := fmt.Sprintf("hub/api/v2/%s/field-sets/datasource", s.ProjectID)
			body := Payload{"fields": []string{"custom2", "custom1"}}
			resp := s.Put(path, body)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusNoContent, resp.StatusCode, "path %s", path)

			path = fmt.Sprintf("hub/api/v2/%s/field-sets/table", s.ProjectID)
			body = Payload{"fields": []string{"custom1"}}
			resp = s.Put(path, body)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusNoContent, resp.StatusCode, "path %s", path)

			path = fmt.Sprintf("hub/api/v2/%s/field-sets/column", s.ProjectID)
			body = Payload{"fields": []string{"custom2"}}
			resp = s.Put(path, body)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusNoContent, resp.StatusCode, "path %s", path)
		})
	})

	s.Run("cleanup", func(s *suite.Suite) {
		if s.Config.SkipCleanup {
			s.Skip("skip datasource cleanup")
		}

		s.Run("delete external datasource", func(s *suite.Suite) {
			path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s", s.ProjectID, externalDS)
			resp := s.Delete(path)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusNoContent, resp.StatusCode)
		})

		s.Run("delete managed datasource", func(s *suite.Suite) {
			s.Run("delete managed table", func(s *suite.Suite) {
				path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s/tables/%s", s.ProjectID, managedDSID, managedTableID)
				resp := s.Delete(path)
				defer resp.Body.Close()
				s.Require().Equal(http.StatusNoContent, resp.StatusCode)

				s.Run("check that we can't query the deleted table", func(s *suite.Suite) {
					path := fmt.Sprintf("/hub/api/v1/%s/query", s.ProjectID)
					resp := s.Post(path, Payload{
						"sql": fmt.Sprintf(`SELECT * FROM %s.%s`, managedDS, managedTable),
					})
					defer resp.Body.Close()
					s.Require().Equal(http.StatusUnprocessableEntity, resp.StatusCode, s.ParsePayload(resp.Body))
				})
			})

			path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s", s.ProjectID, managedDSID)
			resp := s.Delete(path)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusNoContent, resp.StatusCode)
		})

		s.Run("delete the virtual datasource", func(s *suite.Suite) {
			s.Run("delete the view", func(s *suite.Suite) {
				path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s/tables/%s", s.ProjectID, virtualDSID, viewName)
				resp := s.Delete(path)
				defer resp.Body.Close()
				s.Require().Equal(http.StatusNoContent, resp.StatusCode)

				s.Run("check that we can't query the deleted view", func(s *suite.Suite) {
					path := fmt.Sprintf("/hub/api/v1/%s/query", s.ProjectID)
					resp := s.Post(path, Payload{
						"sql": fmt.Sprintf(`SELECT * FROM %s.%s`, virtualDS, viewName),
					})
					defer resp.Body.Close()
					s.Require().Equal(http.StatusUnprocessableEntity, resp.StatusCode, s.ParsePayload(resp.Body))
				})
			})

			path := fmt.Sprintf("/hub/api/v1/%s/datasources/%s", s.ProjectID, virtualDSID)
			resp := s.Delete(path)
			defer resp.Body.Close()
			s.Require().Equal(http.StatusNoContent, resp.StatusCode)
		})

		s.Run("delete test project", func(s *suite.Suite) {
			path := fmt.Sprintf("/auth/api/v2/tenants/%s/realms/%s", s.TenantID, s.ProjectID)
			resp := s.Delete(path)
			defer resp.Body.Close()
			// idp doesn't have 100% proper status codes right now :(
			s.Require().Equal(http.StatusOK, resp.StatusCode)
		})
	})
}
