madlib-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From okis...@apache.org
Subject madlib git commit: Madpack: Add UDO and UDOC automation
Date Wed, 14 Nov 2018 17:17:14 GMT
Repository: madlib
Updated Branches:
  refs/heads/master a61054c31 -> bc8aeeb11


Madpack: Add UDO and UDOC automation

JIRA: MADLIB-1281

- Add scripts for detecting changed/dropped UDOs and UDOCs.
- Expand the create_changelist.py file to consume these scripts and
create changelists with these fields filled if necessary.
- Fix the update_util.py to use the correct dictionary key.
- Add drop operator class command to the svac.sql_in to make sure the
old class is removed before creating the updated one.

Closes #337


Project: http://git-wip-us.apache.org/repos/asf/madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/madlib/commit/bc8aeeb1
Tree: http://git-wip-us.apache.org/repos/asf/madlib/tree/bc8aeeb1
Diff: http://git-wip-us.apache.org/repos/asf/madlib/diff/bc8aeeb1

Branch: refs/heads/master
Commit: bc8aeeb11f9e659dca3f53674260d4c8ad00a882
Parents: a61054c
Author: Orhan Kislal <okislal@apache.org>
Authored: Wed Nov 14 20:15:57 2018 +0300
Committer: Orhan Kislal <okislal@apache.org>
Committed: Wed Nov 14 20:16:30 2018 +0300

----------------------------------------------------------------------
 methods/svec/src/pg_gp/svec.sql_in |   2 +
 src/madpack/create_changelist.py   | 104 ++++++++++++++++++++++++++++++--
 src/madpack/diff_udo.sql           |  81 +++++++++++++++++++++++++
 src/madpack/diff_udoc.sql          |  79 ++++++++++++++++++++++++
 src/madpack/upgrade_util.py        |   4 +-
 5 files changed, 265 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/madlib/blob/bc8aeeb1/methods/svec/src/pg_gp/svec.sql_in
----------------------------------------------------------------------
diff --git a/methods/svec/src/pg_gp/svec.sql_in b/methods/svec/src/pg_gp/svec.sql_in
index 6e9825d..2d0e21e 100644
--- a/methods/svec/src/pg_gp/svec.sql_in
+++ b/methods/svec/src/pg_gp/svec.sql_in
@@ -1188,6 +1188,8 @@ CREATE OPERATOR MADLIB_SCHEMA.*|| (
 	leftarg = int4, rightarg = MADLIB_SCHEMA.svec, procedure = MADLIB_SCHEMA.svec_concat_replicate
 );
 
+DROP OPERATOR CLASS IF EXISTS MADLIB_SCHEMA.svec_ops USING btree;
+
 CREATE OPERATOR CLASS MADLIB_SCHEMA.svec_ops
 DEFAULT FOR TYPE MADLIB_SCHEMA.svec USING btree AS
 OPERATOR        1       MADLIB_SCHEMA.< ,

http://git-wip-us.apache.org/repos/asf/madlib/blob/bc8aeeb1/src/madpack/create_changelist.py
----------------------------------------------------------------------
diff --git a/src/madpack/create_changelist.py b/src/madpack/create_changelist.py
index a689319..a88b71d 100644
--- a/src/madpack/create_changelist.py
+++ b/src/madpack/create_changelist.py
@@ -63,7 +63,16 @@ if err1 != 0:
     raise SystemExit
 
 print "Creating changelist {0}".format(ch_filename)
-os.system("rm -f /tmp/madlib_tmp_nm.txt /tmp/madlib_tmp_udf.txt /tmp/madlib_tmp_udt.txt")
+os.system("""
+    rm -f /tmp/madlib_tmp_nm.txt \
+    /tmp/madlib_tmp_udf.txt \
+    /tmp/madlib_tmp_udt.txt \
+    /tmp/madlib_tmp_udo.txt \
+    /tmp/madlib_tmp_udoc.txt \
+    /tmp/madlib_tmp_typedep.txt \
+    /tmp/madlib_tmp_typedep_udo.txt \
+    /tmp/madlib_tmp_typedep_udoc.txt
+    """)
 try:
     # Find the new modules using the git diff
     err1 = os.system("git diff {old_vers} {new_vers} --name-only --diff-filter=A > /tmp/madlib_tmp_nm.txt".format(**locals()))
@@ -224,8 +233,87 @@ try:
 # List of the UDC, UDO and UDOC changes.
 """)
     f.write("udc:\n")
+
+    # Find the changed operators
+
+    f.write("\n# Changes in the operators (UDO)\n")
     f.write("udo:\n")
+
+    os.system("psql {0} -f diff_udo.sql > /tmp/madlib_tmp_udo.txt".format(database))
+    with open('/tmp/madlib_tmp_udo.txt') as fp:
+        for line in fp:
+            if ' | ' in line:
+                    sp = line.split(' | ')
+
+                    if sp[0].strip() == 'name':
+                        f.write("    - '" + sp[1].strip() + "':\n")
+                    elif sp[0].strip() == 'rettype':
+                        f.write('        rettype: ' + sp[1].strip() + "\n")
+                    elif sp[0].strip() == 'oprright':
+                        f.write('        rightarg: ' + sp[1].strip() + "\n")
+                    elif sp[0].strip() == 'oprleft':
+                        f.write('        leftarg: ' + sp[1].strip() + "\n")
+
+    for t in udt_list:
+
+        os.system("""psql {database} -c "DROP TABLE IF EXISTS __tmp__madlib__ " > /dev/null
""".format(**locals()))
+
+        os.system("""psql {database} -c "SELECT get_udos('__tmp__madlib__', 'madlib_old_vers',
'{t}')" > /dev/null """.format(**locals()))
+
+        os.system("""psql {database} -x -c "SELECT name, rettype, oprright, oprleft FROM
__tmp__madlib__ ORDER BY name DESC" > /tmp/madlib_tmp_typedep_udo.txt """.format(**locals()))
+
+        os.system("""psql {database} -c "DROP TABLE IF EXISTS __tmp__madlib__ " > /dev/null
""".format(**locals()))
+
+        with open('/tmp/madlib_tmp_typedep_udo.txt') as fp:
+            for line in fp:
+                if '|' in line:
+                    sp = line.split('|')
+
+                    if sp[0].strip() == 'name':
+                        f.write("    - '" + sp[1].strip() + "':\n")
+                    elif sp[0].strip() == 'rettype':
+                        f.write('        rettype: ' + sp[1].strip() + "\n")
+                    elif sp[0].strip() == 'oprright':
+                        f.write('        rightarg: ' + sp[1].strip() + "\n")
+                    elif sp[0].strip() == 'oprleft':
+                        f.write('        leftarg: ' + sp[1].strip() + "\n")
+
+
+    # Find the changed operator classes
+
+    f.write("\n# Changes in the operator classes (UDOC)\n")
+    os.system("psql {0} -f diff_udoc.sql > /tmp/madlib_tmp_udoc.txt".format(database))
     f.write("udoc:\n")
+    with open('/tmp/madlib_tmp_udoc.txt') as fp:
+        for line in fp:
+            if '|' in line:
+                    sp = line.split('|')
+
+                    if sp[0].strip() == 'opfamily_name':
+                        f.write("    - '" + sp[1].strip() + "':\n")
+                    elif sp[0].strip() == 'index_method':
+                        f.write('        index_method: ' + sp[1].strip() + "\n")
+
+    for t in udt_list:
+
+        os.system("""psql {database} -c "DROP TABLE IF EXISTS __tmp__madlib__ " > /dev/null
""".format(**locals()))
+
+        os.system("""psql {database} -c "SELECT get_udocs('__tmp__madlib__', 'madlib_old_vers',
'{t}')" > /dev/null """.format(**locals()))
+
+        os.system("""psql {database} -x -c "SELECT opfamily_name, index_method FROM __tmp__madlib__
ORDER BY name DESC" > /tmp/madlib_tmp_typedep_udoc.txt """.format(**locals()))
+
+        os.system("""psql {database} -c "DROP TABLE IF EXISTS __tmp__madlib__ " > /dev/null
""".format(**locals()))
+
+        with open('/tmp/madlib_tmp_typedep_udoc.txt') as fp:
+            for line in fp:
+                if '|' in line:
+                    sp = line.split('|')
+
+                    if sp[0].strip() == 'opfamily_name':
+                        f.write("    - " + sp[1].strip() + ":\n")
+                    elif sp[0].strip() == 'index':
+                        f.write('        index: ' + sp[1].strip() + "\n")
+
     f.close()
 
     # Copy the new changelist file to its proper location
@@ -237,6 +325,14 @@ except:
     print "Something went wrong! The changelist might be wrong/corrupted."
     raise
 finally:
-    os.system("rm -f /tmp/madlib_tmp_nm.txt /tmp/madlib_tmp_udf.txt "
-              "/tmp/madlib_tmp_udt.txt /tmp/madlib_tmp_cl.yaml "
-              "/tmp/madlib_tmp_typedep.txt")
+    os.system("""
+        rm -f /tmp/madlib_tmp_nm.txt \
+        /tmp/madlib_tmp_udf.txt \
+        /tmp/madlib_tmp_udt.txt \
+        /tmp/madlib_tmp_udo.txt \
+        /tmp/madlib_tmp_udoc.txt \
+        /tmp/madlib_tmp_typedep.txt \
+        /tmp/madlib_tmp_typedep_udo.txt \
+        /tmp/madlib_tmp_typedep_udoc.txt \
+        /tmp/madlib_tmp_cl.yaml
+        """)

http://git-wip-us.apache.org/repos/asf/madlib/blob/bc8aeeb1/src/madpack/diff_udo.sql
----------------------------------------------------------------------
diff --git a/src/madpack/diff_udo.sql b/src/madpack/diff_udo.sql
new file mode 100644
index 0000000..affcca9
--- /dev/null
+++ b/src/madpack/diff_udo.sql
@@ -0,0 +1,81 @@
+------------------------------------------------------------------------------
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+
+--   http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+------------------------------------------------------------------------------
+
+SET client_min_messages to ERROR;
+\x on
+
+CREATE OR REPLACE FUNCTION filter_schema(argstr text, schema_name text)
+RETURNS text AS $$
+    if argstr is None:
+        return "NULL"
+    return argstr.replace(schema_name + ".", '')
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION alter_schema(argstr text, schema_name text)
+RETURNS text AS $$
+    if argstr is None:
+        return "NULL"
+    return argstr.replace(schema_name + ".", 'schema_madlib.')
+$$ LANGUAGE plpythonu;
+
+
+CREATE OR REPLACE FUNCTION get_udos(table_name text, schema_name text,
+                                         type_filter text)
+RETURNS VOID AS
+$$
+    import plpy
+
+    plpy.execute("""
+        create table {table_name} AS
+            SELECT *
+            FROM (
+                SELECT n.nspname AS "Schema",
+                       o.oprname AS name,
+                       filter_schema(o.oprcode::text, '{schema_name}') AS oprcode,
+                       alter_schema(pg_catalog.format_type(o.oprleft, NULL), '{schema_name}')
AS oprleft,
+                       alter_schema(pg_catalog.format_type(o.oprright, NULL), '{schema_name}')
AS oprright,
+                       alter_schema(pg_catalog.format_type(o.oprresult, NULL), '{schema_name}')
AS rettype
+                FROM pg_catalog.pg_operator o
+                    LEFT JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace
+                WHERE n.nspname OPERATOR(pg_catalog.~) '^({schema_name})$'
+                ) q
+            WHERE oprleft LIKE 'schema_madlib.{type_filter}'
+                  OR oprleft LIKE 'schema_madlib.{type_filter}[]'
+                  OR oprright LIKE 'schema_madlib.{type_filter}'
+                  OR oprright LIKE 'schema_madlib.{type_filter}[]'
+                  OR rettype LIKE 'schema_madlib.{type_filter}'
+                  OR rettype LIKE 'schema_madlib.{type_filter}[]'
+                  OR '{type_filter}' LIKE 'Full'
+    """.format(table_name=table_name, schema_name=schema_name, type_filter=type_filter))
+$$ LANGUAGE plpythonu;
+
+
+DROP TABLE if exists udo_madlib_old_version;
+DROP TABLE if exists udo_madlib_new_version;
+
+SELECT get_udos('udo_madlib_old_version','madlib_old_vers','Full');
+SELECT get_udos('udo_madlib_new_version','madlib','Full');
+
+
+SELECT old.name AS name , old.oprright AS oprright,
+       old.oprleft AS oprleft, old.rettype AS rettype
+FROM udo_madlib_old_version AS old LEFT JOIN udo_madlib_new_version
+    USING (name, oprcode, oprright, oprleft, rettype)
+WHERE udo_madlib_new_version.name is NULL
+ORDER BY old.name;

http://git-wip-us.apache.org/repos/asf/madlib/blob/bc8aeeb1/src/madpack/diff_udoc.sql
----------------------------------------------------------------------
diff --git a/src/madpack/diff_udoc.sql b/src/madpack/diff_udoc.sql
new file mode 100644
index 0000000..c1312dc
--- /dev/null
+++ b/src/madpack/diff_udoc.sql
@@ -0,0 +1,79 @@
+------------------------------------------------------------------------------
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+
+--   http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+------------------------------------------------------------------------------
+
+SET client_min_messages to ERROR;
+\x on
+
+
+CREATE OR REPLACE FUNCTION filter_schema(argstr text, schema_name text)
+RETURNS text AS $$
+    if argstr is None:
+        return "NULL"
+    return argstr.replace(schema_name + ".", '')
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION alter_schema(argstr text, schema_name text)
+RETURNS text AS $$
+    if argstr is None:
+        return "NULL"
+    return argstr.replace(schema_name + ".", 'schema_madlib.')
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION get_udocs(table_name text, schema_name text,
+                                         type_filter text)
+RETURNS VOID AS
+$$
+    import plpy
+
+    plpy.execute("""
+	CREATE TABLE {table_name} AS
+	SELECT * FROM (
+		SELECT index_method, opfamily_name,
+			   array_to_string(array_agg(alter_schema(opfamily_operator::text, '{schema_name}')),
',')
+			   	AS operators
+		FROM (
+			SELECT am.amname AS index_method, opf.opfname AS opfamily_name,
+		       	   amop.amopopr::regoperator AS opfamily_operator
+		    FROM pg_am am, pg_opfamily opf, pg_amop amop, pg_namespace n
+		    WHERE opf.opfmethod = am.oid AND
+		          amop.amopfamily = opf.oid AND
+		          n.oid = opf.opfnamespace AND
+		          n.nspname OPERATOR(pg_catalog.~) '^({schema_name})$'
+		    ORDER BY index_method, opfamily_name, opfamily_operator
+		    ) q
+		GROUP BY (index_method, opfamily_name)
+		) qq
+	WHERE operators LIKE '%schema_madlib.{type_filter}%' OR '{type_filter}' LIKE 'Full'
+	""".format(table_name=table_name, schema_name=schema_name, type_filter=type_filter))
+
+$$ LANGUAGE plpythonu;
+
+DROP TABLE if exists udoc_madlib_old_version;
+DROP TABLE if exists udoc_madlib_new_version;
+
+SELECT get_udocs('udoc_madlib_old_version','madlib_old_vers','Full');
+SELECT get_udocs('udoc_madlib_new_version','madlib','Full');
+
+
+SELECT old.opfamily_name, old.index_method
+FROM udoc_madlib_old_version AS old LEFT JOIN udoc_madlib_new_version
+	USING (index_method, opfamily_name, operators)
+WHERE udoc_madlib_new_version.opfamily_name is NULL
+ORDER BY 1;
+

http://git-wip-us.apache.org/repos/asf/madlib/blob/bc8aeeb1/src/madpack/upgrade_util.py
----------------------------------------------------------------------
diff --git a/src/madpack/upgrade_util.py b/src/madpack/upgrade_util.py
index e50e7b9..90511a3 100644
--- a/src/madpack/upgrade_util.py
+++ b/src/madpack/upgrade_util.py
@@ -214,6 +214,8 @@ class ChangeHandler(UpgradeBase):
         self._add_to_dict(config['udc'], self._udc)
         self._add_to_dict(self._load_config_param(config['udf']), self._udf)
         self._add_to_dict(self._load_config_param(config['uda']), self._uda)
+        self._add_to_dict(self._load_config_param(config['udo']), self._udo)
+        self._add_to_dict(self._load_config_param(config['udoc']), self._udoc)
 
     def _get_relevant_filenames(self, upgrade_from):
         """ Get all changelist files that together describe the upgrade process
@@ -473,7 +475,7 @@ class ChangeHandler(UpgradeBase):
         """
         for op_cls in self._udoc:
             for value in self._udoc[op_cls]:
-                index = value['index']
+                index = value['index_method']
                 _write_to_file(self.output_filehandle, """
                     DROP OPERATOR CLASS IF EXISTS {schema}.{op_cls} USING {index};
                     """.format(schema=self._schema, **locals()))


Mime
View raw message