Add index and table metrics to vttablet

changelog/22.0/22.0.0/summary.md

@@ -12,6 +12,7 @@
     - [gh-ost and pt-osc Online DDL strategies](#deleted-ghost-ptosc)
   - **[RPC Changes](#rpc-changes)**
   - **[New VTGate Metrics](#new-vtgate-metrics)**
+  - **[New VTTablet Metrics](#new-vtgate-metrics)**
   - **[Prefer not promoting a replica that is currently taking a backup](#reparents-prefer-not-backing-up)**
   - **[VTOrc Config File Changes](#vtorc-config-file-changes)**
   - **[VTGate Config File Changes](#vtgate-config-file-changes)**
@@ -125,6 +126,16 @@ Via this work we have deprecated several vtgate metrics, please see the [Depreca
 
 ---
 
+### <a id="new-vttablet-metrics"/>New VTTablet Metrics
+
+Four new metrics gauge have been introduced to VTTablet:
+1.	`TableRows` – The estimated number of rows in the table. **Dimensions:** Table.
+2.	`TableClusteredIndexSize` – The byte size of the clustered index (i.e. row data). **Dimensions:** Table.
+3.	`IndexCardinality` – The estimated number of unique values in the index. **Dimensions:** Table, Index. 
+4.	`IndexBytes` – The byte size of the index. **Dimensions:** Table, Index. 
+
+---
+
 ### <a id="reparents-prefer-not-backing-up"/>Prefer not promoting a replica that is currently taking a backup
 
 Emergency reparents now prefer not promoting replicas that are currently taking backups with a backup engine other than

go/mysql/flavor.go

@@ -152,6 +152,10 @@ type flavor interface {
 	baseShowTables() string
 	baseShowTablesWithSizes() string
 	baseShowInnodbTableSizes() string
+	baseShowPartitions() string
+	baseShowTableRowCountClusteredIndex() string
+	baseShowIndexSizes() string
+	baseShowIndexCardinalities() string
 
 	supportsCapability(capability capabilities.FlavorCapability) (bool, error)
 }
@@ -460,6 +464,22 @@ func (c *Conn) BaseShowInnodbTableSizes() string {
 	return c.flavor.baseShowInnodbTableSizes()
 }
 
+func (c *Conn) BaseShowPartitions() string {
+	return c.flavor.baseShowPartitions()
+}
+
+func (c *Conn) BaseShowTableRowCountClusteredIndex() string {
+	return c.flavor.baseShowTableRowCountClusteredIndex()
+}
+
+func (c *Conn) BaseShowIndexSizes() string {
+	return c.flavor.baseShowIndexSizes()
+}
+
+func (c *Conn) BaseShowIndexCardinalities() string {
+	return c.flavor.baseShowIndexCardinalities()
+}
+
 // SupportsCapability checks if the database server supports the given capability
 func (c *Conn) SupportsCapability(capability capabilities.FlavorCapability) (bool, error) {
 	return c.flavor.supportsCapability(capability)

go/mysql/flavor_filepos.go

@@ -370,6 +370,22 @@ func (filePosFlavor) baseShowInnodbTableSizes() string {
 	return ""
 }
 
+func (filePosFlavor) baseShowPartitions() string {
+	return ""
+}
+
+func (filePosFlavor) baseShowTableRowCountClusteredIndex() string {
+	return ""
+}
+
+func (filePosFlavor) baseShowIndexSizes() string {
+	return ""
+}
+
+func (filePosFlavor) baseShowIndexCardinalities() string {
+	return ""
+}
+
 // supportsCapability is part of the Flavor interface.
 func (f *filePosFlavor) supportsCapability(capability capabilities.FlavorCapability) (bool, error) {
 	switch capability {

go/mysql/flavor_mariadb_binlog_playback.go

@@ -26,6 +26,22 @@ func (mariadbFlavor) baseShowInnodbTableSizes() string {
 	return ""
 }
 
+func (mariadbFlavor) baseShowPartitions() string {
+	return ""
+}
+
+func (mariadbFlavor) baseShowTableRowCountClusteredIndex() string {
+	return ""
+}
+
+func (mariadbFlavor) baseShowIndexSizes() string {
+	return ""
+}
+
+func (mariadbFlavor) baseShowIndexCardinalities() string {
+	return ""
+}
+
 // baseShowTablesWithSizes is part of the Flavor interface.
 func (mariadbFlavor101) baseShowTablesWithSizes() string {
 	return TablesWithSize56

go/mysql/flavor_mysql.go

@@ -485,6 +485,11 @@ const InnoDBTableSizes = `
 		GROUP BY it.name
 `
 
+const ShowPartitons = `select table_name, partition_name from information_schema.partitions where table_schema = database() and partition_name is not null`
+const ShowTableRowCountClusteredIndex = `select table_name, n_rows, clustered_index_size * @@innodb_page_size from mysql.innodb_table_stats where database_name = database()`
+const ShowIndexSizes = `select table_name, index_name, stat_value * @@innodb_page_size from mysql.innodb_index_stats where database_name = database() and stat_name = 'size'`
+const ShowIndexCardinalities = `select table_name, index_name, max(cardinality) from information_schema.statistics s where table_schema = database() group by s.table_name, s.index_name`
+
 // baseShowTablesWithSizes is part of the Flavor interface.
 func (mysqlFlavor57) baseShowTablesWithSizes() string {
 	// For 5.7, we use the base query instead of the query with sizes. Flavor57 should only be used
@@ -500,6 +505,22 @@ func (mysqlFlavor57) baseShowInnodbTableSizes() string {
 	return ""
 }
 
+func (mysqlFlavor57) baseShowPartitions() string {
+	return ""
+}
+
+func (mysqlFlavor57) baseShowTableRowCountClusteredIndex() string {
+	return ""
+}
+
+func (mysqlFlavor57) baseShowIndexSizes() string {
+	return ""
+}
+
+func (mysqlFlavor57) baseShowIndexCardinalities() string {
+	return ""
+}
+
 // supportsCapability is part of the Flavor interface.
 func (f mysqlFlavor) supportsCapability(capability capabilities.FlavorCapability) (bool, error) {
 	return capabilities.MySQLVersionHasCapability(f.serverVersion, capability)
@@ -515,6 +536,22 @@ func (mysqlFlavor) baseShowInnodbTableSizes() string {
 	return InnoDBTableSizes
 }
 
+func (mysqlFlavor) baseShowPartitions() string {
+	return ShowPartitons
+}
+
+func (mysqlFlavor) baseShowTableRowCountClusteredIndex() string {
+	return ShowTableRowCountClusteredIndex
+}
+
+func (mysqlFlavor) baseShowIndexSizes() string {
+	return ShowIndexSizes
+}
+
+func (mysqlFlavor) baseShowIndexCardinalities() string {
+	return ShowIndexCardinalities
+}
+
 func (mysqlFlavor) setReplicationSourceCommand(params *ConnParams, host string, port int32, heartbeatInterval float64, connectRetry int) string {
 	args := []string{
 		fmt.Sprintf("SOURCE_HOST = '%s'", host),

go/vt/vttablet/endtoend/misc_test.go

@@ -1090,6 +1090,66 @@ func TestEngineReload(t *testing.T) {
 	})
 }
 
+func TestUpdateTableIndexMetrics(t *testing.T) {
+	ctx := context.Background()
+	conn, err := mysql.Connect(ctx, &connParams)
+	require.NoError(t, err)
+	defer conn.Close()
+
+	if query := conn.BaseShowInnodbTableSizes(); query == "" {
+		t.Skip("additional table/index metrics not updated in this version of MySQL")
+	}
+	client := framework.NewClient()
+
+	_, err = client.Execute("insert into vitess_part (id) values (5),(15),(25)", nil)
+	require.NoError(t, err)
+	defer client.Execute("delete from vitess_part where id in (5,15,25)", nil)
+
+	// Analyze tables to make sure stats are updated prior to reload
+	tables := []string{"vitess_a", "vitess_part", "vitess_autoinc_seq"}
+	for _, table := range tables {
+		_, err = client.Execute(fmt.Sprintf("analyze table %s", table), nil)
+		require.NoError(t, err)
+	}
+
+	// Wait up to 5s for the rows added to vitess_part to be reflected in DebugVars
+	updated := false
+	for i := 0; !updated && i < 10; i++ {
+		err = framework.Server.ReloadSchema(ctx)
+		require.NoError(t, err)
+
+		if framework.FetchVal(framework.DebugVars(), "TableRows/vitess_part") == 3 {
+			updated = true
+		} else {
+			time.Sleep(500 * time.Millisecond)
+		}
+	}
+
+	results, err := client.Execute("select @@innodb_page_size", nil)
+	require.NoError(t, err)
+	pageSize, err := results.Rows[0][0].ToFloat64()
+	require.NoError(t, err)
+
+	vars := framework.DebugVars()
+
+	assert.Equal(t, 2.0, framework.FetchVal(vars, "TableRows/vitess_a"))
+	assert.Equal(t, 3.0, framework.FetchVal(vars, "TableRows/vitess_part"))
+	partTableCountResult, _ := client.Execute("select count(1) from vitess_part", nil)
+	partTableRows, _ := partTableCountResult.Rows[0][0].ToInt()
+	assert.Equal(t, 3, partTableRows)
+
+	assert.Equal(t, pageSize, framework.FetchVal(vars, "TableClusteredIndexSize/vitess_a"))
+	assert.Equal(t, pageSize*2, framework.FetchVal(vars, "TableClusteredIndexSize/vitess_part"))
+
+	assert.Equal(t, 2.0, framework.FetchVal(vars, "IndexCardinality/vitess_a.PRIMARY"))
+	assert.Equal(t, 3.0, framework.FetchVal(vars, "IndexCardinality/vitess_part.PRIMARY"))
+	assert.Equal(t, 0.0, framework.FetchVal(vars, "IndexCardinality/vitess_autoinc_seq.name"))
+
+	assert.Equal(t, pageSize, framework.FetchVal(vars, "IndexBytes/vitess_a.PRIMARY"))
+	assert.Equal(t, pageSize*2, framework.FetchVal(vars, "IndexBytes/vitess_part.PRIMARY"))
+	assert.Equal(t, pageSize, framework.FetchVal(vars, "IndexBytes/vitess_autoinc_seq.name"))
+}
+
 // TestTuple tests that bind variables having tuple values work with vttablet.
 func TestTuple(t *testing.T) {
 	client := framework.NewClient()

go/vt/vttablet/tabletserver/connpool/dbconn.go

@@ -568,6 +568,22 @@ func (dbc *Conn) BaseShowInnodbTableSizes() string {
 	return dbc.conn.BaseShowInnodbTableSizes()
 }
 
+func (dbc *Conn) BaseShowPartitions() string {
+	return dbc.conn.BaseShowPartitions()
+}
+
+func (dbc *Conn) BaseShowTableRowCountClusteredIndex() string {
+	return dbc.conn.BaseShowTableRowCountClusteredIndex()
+}
+
+func (dbc *Conn) BaseShowIndexSizes() string {
+	return dbc.conn.BaseShowIndexSizes()
+}
+
+func (dbc *Conn) BaseShowIndexCardinalities() string {
+	return dbc.conn.BaseShowIndexCardinalities()
+}
+
 func (dbc *Conn) ConnCheck(ctx context.Context) error {
 	if err := dbc.conn.ConnCheck(); err != nil {
 		return dbc.Reconnect(ctx)

go/vt/vttablet/tabletserver/health_streamer_test.go

@@ -404,6 +404,10 @@ func TestReloadView(t *testing.T) {
 	db.AddQuery("SELECT TABLE_NAME, CREATE_TIME FROM _vt.`tables`", &sqltypes.Result{})
 	// adding query pattern for udfs
 	db.AddQueryPattern("SELECT name.*", &sqltypes.Result{})
+	db.AddQuery(mysql.ShowPartitons, &sqltypes.Result{})
+	db.AddQuery(mysql.ShowTableRowCountClusteredIndex, &sqltypes.Result{})
+	db.AddQuery(mysql.ShowIndexSizes, &sqltypes.Result{})
+	db.AddQuery(mysql.ShowIndexCardinalities, &sqltypes.Result{})
 
 	se.InitDBConfig(cfg.DB.DbaWithDB())
 	hs.Open()