trafficcontrol-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From raw...@apache.org
Subject [trafficcontrol] branch 4.0.x updated: Fix atscfg meta to omit config files for nonexistent delivery services (#4256) (#4278)
Date Thu, 09 Jan 2020 19:44:41 GMT
This is an automated email from the ASF dual-hosted git repository.

rawlin pushed a commit to branch 4.0.x
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git


The following commit(s) were added to refs/heads/4.0.x by this push:
     new c085104  Fix atscfg meta to omit config files for nonexistent delivery services (#4256)
(#4278)
c085104 is described below

commit c085104164fbc0f0adc6e049e242106fccae3159
Author: Robert Butts <rob05c@users.noreply.github.com>
AuthorDate: Thu Jan 9 12:44:32 2020 -0700

    Fix atscfg meta to omit config files for nonexistent delivery services (#4256) (#4278)
    
    * Fix atscfg meta to omit nonexistent ds files
    
    This fixes atscfg to omit files with location parameters for config
    files, hdr_rw_, regex_remap_, url_sig_, and uri_signing_ for DSes
    which don't exist (typically after deleting a DS but forgetting to
    delete the corresponding location Parameters).
    
    This is important to make atstccfg work because without it, ORT will
    request url_sig_ files for DSes which don't exist, and the API
    endpoint to get the keys will fail. Because atstccfg uses the API,
    there's no reasonable way to make it work, except to fix the meta
    config to omit those nonexistent files.
    
    In addition to nonexistent files, this also omits files for DSes
    not assigned to the server. There won't even be remap lines for
    those DSes anyway, so it's better and safer to omit the ancillary
    files altogether.
    
    * Fix TO API test for meta cfg to set DSS, real DS
    
    (cherry picked from commit ea315f079c0142169069e4705a66c5a70f0bbb56)
---
 CHANGELOG.md                                       |  1 +
 lib/go-atscfg/meta.go                              | 20 +++++++++++++++
 lib/go-atscfg/meta_test.go                         | 29 +++++++++++++++++++--
 traffic_ops/ort/atstccfg/cfgfile/meta.go           | 14 +++++-----
 traffic_ops/testing/api/v14/atsconfig_meta_test.go |  6 +++--
 .../traffic_ops_golang/ats/atsserver/meta.go       |  8 +++++-
 traffic_ops/traffic_ops_golang/ats/db.go           | 30 ++++++++++++++++++++++
 7 files changed, 95 insertions(+), 13 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d37e64b..7b4cecd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -64,6 +64,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Traffic Ops now supports a "sortOrder" query parameter on some endpoints to return API
responses in descending order
 - Traffic Ops now uses a consistent format for audit logs across all Go endpoints
 - Added cache-side config generator, atstccfg, installed with ORT. Includes all configs.
Includes a plugin system.
+- Fixed ATS config generation to omit regex remap, header rewrite, URL Sig, and URI Signing
files for delivery services not assigned to that server.
 - In Traffic Portal, all tables now include a 'CSV' link to enable the export of table data
in CSV format.
 - Pylint configuration now enforced (present in [a file in the Python client directory](./traffic_control/clients/python/pylint.rc))
 - Added an optional SMTP server configuration to the TO configuration file, api now has unused
abilitiy to send emails
diff --git a/lib/go-atscfg/meta.go b/lib/go-atscfg/meta.go
index c8f58e6..fbcb841 100644
--- a/lib/go-atscfg/meta.go
+++ b/lib/go-atscfg/meta.go
@@ -48,6 +48,7 @@ func MakeMetaConfig(
 	locationParams map[string]ConfigProfileParams, // map[configFile]params; 'location' and
'URL' Parameters on serverHostName's Profile
 	uriSignedDSes []tc.DeliveryServiceName,
 	scopeParams map[string]string, // map[configFileName]scopeParam
+	dsNames map[tc.DeliveryServiceName]struct{},
 ) string {
 	if tmURL == "" {
 		log.Errorln("ats.GetConfigMetadata: global tm.url parameter missing or empty! Setting empty
in meta config!")
@@ -83,7 +84,26 @@ func MakeMetaConfig(
 		}
 	}
 
+locationParamsFor:
 	for cfgFile, cfgParams := range locationParams {
+		if strings.HasSuffix(cfgFile, ".config") {
+			dsConfigFilePrefixes := []string{
+				"hdr_rw_",
+				"regex_remap_",
+				"url_sig_",
+				"uri_signing_",
+			}
+			for _, prefix := range dsConfigFilePrefixes {
+				if strings.HasPrefix(cfgFile, prefix) {
+					dsName := strings.TrimSuffix(strings.TrimPrefix(cfgFile, prefix), ".config")
+					if _, ok := dsNames[tc.DeliveryServiceName(dsName)]; !ok {
+						log.Warnln("Server Profile had 'location' Parameter '" + cfgFile + "', but delivery
Service '" + dsName + "' is not assigned to this Server! Not including in meta config!")
+						continue locationParamsFor
+					}
+				}
+			}
+		}
+
 		atsCfg := tc.ATSConfigMetaDataConfigFile{
 			FileNameOnDisk: cfgParams.FileNameOnDisk,
 			Location:       cfgParams.Location,
diff --git a/lib/go-atscfg/meta_test.go b/lib/go-atscfg/meta_test.go
index fed1549..5317220 100644
--- a/lib/go-atscfg/meta_test.go
+++ b/lib/go-atscfg/meta_test.go
@@ -21,6 +21,7 @@ package atscfg
 
 import (
 	"encoding/json"
+	"strings"
 	"testing"
 
 	"github.com/apache/trafficcontrol/lib/go-tc"
@@ -80,6 +81,26 @@ func TestMakeMetaConfig(t *testing.T) {
 			FileNameOnDisk: "uri_signing_mydsname.config",
 			Location:       "/my/location/",
 		},
+		"uri_signing_nonexistentds.config": ConfigProfileParams{
+			FileNameOnDisk: "uri_signing_nonexistentds.config",
+			Location:       "/my/location/",
+		},
+		"regex_remap_nonexistentds.config": ConfigProfileParams{
+			FileNameOnDisk: "regex_remap_nonexistentds.config",
+			Location:       "/my/location/",
+		},
+		"url_sig_nonexistentds.config": ConfigProfileParams{
+			FileNameOnDisk: "url_sig_nonexistentds.config",
+			Location:       "/my/location/",
+		},
+		"hdr_rw_nonexistentds.config": ConfigProfileParams{
+			FileNameOnDisk: "hdr_rw_nonexistentds.config",
+			Location:       "/my/location/",
+		},
+		"hdr_rw_mid_nonexistentds.config": ConfigProfileParams{
+			FileNameOnDisk: "hdr_rw_mid_nonexistentds.config",
+			Location:       "/my/location/",
+		},
 		"unknown.config": ConfigProfileParams{
 			FileNameOnDisk: "unknown.config",
 			Location:       "/my/location/",
@@ -90,10 +111,11 @@ func TestMakeMetaConfig(t *testing.T) {
 		},
 	}
 	uriSignedDSes := []tc.DeliveryServiceName{"mydsname"}
+	dses := map[tc.DeliveryServiceName]struct{}{"mydsname": {}}
 
 	scopeParams := map[string]string{"custom.config": string(tc.ATSConfigMetaDataConfigFileScopeProfiles)}
 
-	txt := MakeMetaConfig(serverHostName, server, tmURL, tmReverseProxyURL, locationParams,
uriSignedDSes, scopeParams)
+	txt := MakeMetaConfig(serverHostName, server, tmURL, tmReverseProxyURL, locationParams,
uriSignedDSes, scopeParams, dses)
 
 	cfg := tc.ATSConfigMetaData{}
 	if err := json.Unmarshal([]byte(txt), &cfg); err != nil {
@@ -223,7 +245,7 @@ func TestMakeMetaConfig(t *testing.T) {
 	}
 
 	server.Type = "MID"
-	txt = MakeMetaConfig(serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes,
scopeParams)
+	txt = MakeMetaConfig(serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes,
scopeParams, dses)
 	cfg = tc.ATSConfigMetaData{}
 	if err := json.Unmarshal([]byte(txt), &cfg); err != nil {
 		t.Fatalf("MakeMetaConfig returned invalid JSON: " + err.Error())
@@ -237,4 +259,7 @@ func TestMakeMetaConfig(t *testing.T) {
 		}
 		break
 	}
+	if strings.Contains(txt, "nonexistentds") {
+		t.Errorf("expected location parameters for nonexistent delivery services to not be added
to config, actual '%v'", txt)
+	}
 }
diff --git a/traffic_ops/ort/atstccfg/cfgfile/meta.go b/traffic_ops/ort/atstccfg/cfgfile/meta.go
index d8f3f56..18dd27e 100644
--- a/traffic_ops/ort/atstccfg/cfgfile/meta.go
+++ b/traffic_ops/ort/atstccfg/cfgfile/meta.go
@@ -185,9 +185,6 @@ func GetConfigFileMeta(cfg config.TCCfg, serverNameOrID string) (string,
error)
 
 	dsIDs := []int{}
 	for _, ds := range deliveryServices {
-		if ds.SigningAlgorithm == nil || *ds.SigningAlgorithm != tc.SigningAlgorithmURISigning
{
-			continue
-		}
 		if ds.ID == nil {
 			// TODO log error?
 			continue
@@ -210,6 +207,7 @@ func GetConfigFileMeta(cfg config.TCCfg, serverNameOrID string) (string,
error)
 		dssMap[*dss.DeliveryService] = struct{}{}
 	}
 
+	dsNames := map[tc.DeliveryServiceName]struct{}{}
 	uriSignedDSes := []tc.DeliveryServiceName{}
 	for _, ds := range deliveryServices {
 		if ds.ID == nil {
@@ -218,14 +216,14 @@ func GetConfigFileMeta(cfg config.TCCfg, serverNameOrID string) (string,
error)
 		if ds.XMLID == nil {
 			continue // TODO log?
 		}
-		if ds.SigningAlgorithm == nil || *ds.SigningAlgorithm != tc.SigningAlgorithmURISigning
{
-			continue
-		}
 		if _, ok := dssMap[*ds.ID]; !ok {
 			continue
 		}
-		uriSignedDSes = append(uriSignedDSes, tc.DeliveryServiceName(*ds.XMLID))
+		dsNames[tc.DeliveryServiceName(*ds.XMLID)] = struct{}{}
+		if ds.SigningAlgorithm != nil && *ds.SigningAlgorithm == tc.SigningAlgorithmURISigning
{
+			uriSignedDSes = append(uriSignedDSes, tc.DeliveryServiceName(*ds.XMLID))
+		}
 	}
 
-	return atscfg.MakeMetaConfig(tc.CacheName(serverHostName), &serverInfo, toURL, toReverseProxyURL,
locationParams, uriSignedDSes, scopeParams), nil
+	return atscfg.MakeMetaConfig(tc.CacheName(serverHostName), &serverInfo, toURL, toReverseProxyURL,
locationParams, uriSignedDSes, scopeParams, dsNames), nil
 }
diff --git a/traffic_ops/testing/api/v14/atsconfig_meta_test.go b/traffic_ops/testing/api/v14/atsconfig_meta_test.go
index cc8bb39..378c797 100644
--- a/traffic_ops/testing/api/v14/atsconfig_meta_test.go
+++ b/traffic_ops/testing/api/v14/atsconfig_meta_test.go
@@ -24,6 +24,8 @@ import (
 
 func TestATSConfigMeta(t *testing.T) {
 	WithObjs(t, []TCObj{CDNs, Types, Tenants, Parameters, Profiles, Statuses, Divisions, Regions,
PhysLocations, CacheGroups, Servers, DeliveryServices}, func() {
+		defer DeleteTestDeliveryServiceServersCreated(t)
+		CreateTestDeliveryServiceServers(t)
 		GetTestATSConfigMeta(t)
 	})
 }
@@ -49,9 +51,9 @@ func GetTestATSConfigMeta(t *testing.T) {
 	}
 
 	expected := tc.ATSConfigMetaDataConfigFile{
-		FileNameOnDisk: "hdr_rw_mid_anymap-ds.config",
+		FileNameOnDisk: "hdr_rw_ds1.config",
 		Location:       "/remap/config/location/parameter",
-		APIURI:         "cdns/cdn1/configfiles/ats/hdr_rw_mid_anymap-ds.config", // expected suffix;
config gen doesn't care about API version
+		APIURI:         "cdns/cdn1/configfiles/ats/hdr_rw_ds1.config", // expected suffix; config
gen doesn't care about API version
 		URL:            "",
 		Scope:          "cdns",
 	}
diff --git a/traffic_ops/traffic_ops_golang/ats/atsserver/meta.go b/traffic_ops/traffic_ops_golang/ats/atsserver/meta.go
index 65af586..26e741d 100644
--- a/traffic_ops/traffic_ops_golang/ats/atsserver/meta.go
+++ b/traffic_ops/traffic_ops_golang/ats/atsserver/meta.go
@@ -76,7 +76,13 @@ func GetConfigMetaData(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	txt := atscfg.MakeMetaConfig(serverName, server, tmParams.URL, tmParams.ReverseProxyURL,
locationParams, uriSignedDSes, scopeParams)
+	dsNames, err := ats.GetDSNames(inf.Tx.Tx, server.HostName, server.Port)
+	if err != nil {
+		api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("GetConfigMetaData
getting server ds names: "+err.Error()))
+		return
+	}
+
+	txt := atscfg.MakeMetaConfig(serverName, server, tmParams.URL, tmParams.ReverseProxyURL,
locationParams, uriSignedDSes, scopeParams, dsNames)
 	w.Header().Set(rfc.ContentType, rfc.ApplicationJSON)
 	w.Write([]byte(txt))
 }
diff --git a/traffic_ops/traffic_ops_golang/ats/db.go b/traffic_ops/traffic_ops_golang/ats/db.go
index baa030c..26fe44b 100644
--- a/traffic_ops/traffic_ops_golang/ats/db.go
+++ b/traffic_ops/traffic_ops_golang/ats/db.go
@@ -749,6 +749,36 @@ WHERE
 	return dses, nil
 }
 
+// GetDSNames returns a set of delivery service names which have the given server assigned,
and any error.
+func GetDSNames(tx *sql.Tx, serverHostName string, serverPort int) (map[tc.DeliveryServiceName]struct{},
error) {
+	qry := `
+SELECT
+  ds.xml_id
+FROM
+  deliveryservice ds
+  JOIN deliveryservice_server dss ON ds.id = dss.deliveryservice
+  JOIN server s ON s.id = dss.server
+WHERE
+  s.host_name = $1
+  AND s.tcp_port = $2
+`
+	rows, err := tx.Query(qry, serverHostName, serverPort)
+	if err != nil {
+		return nil, errors.New("querying: " + err.Error())
+	}
+	defer rows.Close()
+
+	dses := map[tc.DeliveryServiceName]struct{}{}
+	for rows.Next() {
+		ds := tc.DeliveryServiceName("")
+		if err := rows.Scan(&ds); err != nil {
+			return nil, errors.New("scanning: " + err.Error())
+		}
+		dses[ds] = struct{}{}
+	}
+	return dses, nil
+}
+
 type TMParams struct {
 	URL             string
 	ReverseProxyURL string


Mime
View raw message