From a3357ab8480b2a7b855386b8287aea954e074fa2 Mon Sep 17 00:00:00 2001 From: Manuel Date: Sun, 9 Dec 2018 23:28:07 -0400 Subject: [PATCH 01/24] Commenting triggers and udt_general_tables_insert.sql --- execute_all.sh | 2 +- .../temporal_schema/tables/tbl_categories.sql | 3 ++- .../temporal_schema/tables/tbl_endangered_animals.sql | 5 +++-- .../temporal_schema/tables/tbl_reasons.sql | 3 ++- .../tables/tbl_relation_endangered_reasons.sql | 3 ++- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/execute_all.sh b/execute_all.sh index 2e41c71..c7343bb 100755 --- a/execute_all.sh +++ b/execute_all.sh @@ -3,7 +3,7 @@ # sudo -i -u postgres psql -U postgres -a -f ./postgresql/main_database.sql psql -U postgres -d dev_postgres_database -a -c "CREATE SCHEMA temporal_schema;" -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udt_general_tables_insert.sql +#psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udt_general_tables_insert.sql psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/tables/tbl_categories.sql psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_categories.sql psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/tables/tbl_endangered_animals.sql diff --git a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_categories.sql b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_categories.sql index c3aac0d..a5336a8 100644 --- a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_categories.sql +++ b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_categories.sql @@ -6,6 +6,7 @@ CREATE TABLE temporal_schema.tbl_categories ( is_deleted BOOLEAN DEFAULT false, last_modified_date timestamp with time zone NOT NULL ); -CREATE TRIGGER trigger_categories_insert BEFORE UPDATE OR DELETE ON temporal_schema.tbl_categories +/*CREATE TRIGGER trigger_categories_insert BEFORE UPDATE OR DELETE ON temporal_schema.tbl_categories FOR EACH ROW EXECUTE PROCEDURE udt_general_tables_insert(); +*/ \ No newline at end of file diff --git a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_endangered_animals.sql b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_endangered_animals.sql index d845261..e17df9e 100644 --- a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_endangered_animals.sql +++ b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_endangered_animals.sql @@ -9,6 +9,7 @@ CREATE TABLE temporal_schema.tbl_endangered_animals ( created_date timestamp with time zone DEFAULT NOW(), PRIMARY KEY (id_endangered_animals,id_categories) ); -CREATE TRIGGER trigger_endangered_animals_insert BEFORE UPDATE OR DELETE ON temporal_schema.tbl_endangered_animals +/*CREATE TRIGGER trigger_endangered_animals_insert BEFORE UPDATE OR DELETE ON temporal_schema.tbl_endangered_animals FOR EACH ROW - EXECUTE PROCEDURE udt_general_tables_insert(); \ No newline at end of file + EXECUTE PROCEDURE udt_general_tables_insert(); +*/ \ No newline at end of file diff --git a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_reasons.sql b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_reasons.sql index e45d52f..b256e3f 100644 --- a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_reasons.sql +++ b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_reasons.sql @@ -6,6 +6,7 @@ CREATE TABLE temporal_schema.tbl_reasons ( is_deleted BOOLEAN DEFAULT false, last_modified_date timestamp with time zone NOT NULL ); -CREATE TRIGGER trigger_reasons_insert BEFORE UPDATE OR DELETE ON temporal_schema.tbl_reasons +/*CREATE TRIGGER trigger_reasons_insert BEFORE UPDATE OR DELETE ON temporal_schema.tbl_reasons FOR EACH ROW EXECUTE PROCEDURE udt_general_tables_insert(); +*/ \ No newline at end of file diff --git a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_relation_endangered_reasons.sql b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_relation_endangered_reasons.sql index f16a7fe..71f8555 100644 --- a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_relation_endangered_reasons.sql +++ b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_relation_endangered_reasons.sql @@ -16,6 +16,7 @@ ON UPDATE CASCADE; ALTER TABLE temporal_schema.tbl_relation_endangered_reasons ADD CONSTRAINT tbl_relation_endangered_reasons_tbl_endangered_animals FOREIGN KEY (id_endangered_animals,id_categories) REFERENCES temporal_schema.tbl_endangered_animals (id_endangered_animals,id_categories) ON UPDATE CASCADE; -CREATE TRIGGER trigger_relation_endangered_reasons_insert BEFORE UPDATE OR DELETE ON temporal_schema.tbl_relation_endangered_reasons +/*CREATE TRIGGER trigger_relation_endangered_reasons_insert BEFORE UPDATE OR DELETE ON temporal_schema.tbl_relation_endangered_reasons FOR EACH ROW EXECUTE PROCEDURE udt_general_tables_insert(); +*/ \ No newline at end of file From e66bbf0579d831b2411e2dd804744790349bb395 Mon Sep 17 00:00:00 2001 From: Manuel Date: Thu, 28 Mar 2019 21:42:52 -0400 Subject: [PATCH 02/24] Adding udf_adjust_sequence to order the values of all the sequences, to the last value --- .../public/functions/udf_adjust_sequence.sql | 60 +++++++++++++++ .../public/functions/udf_check_value.sql | 33 ++++++++ .../functions/udf_kill_connections_pg3.sql | 76 +++++++++++++++++++ .../functions/udf_kill_connections_pg4.sql | 76 +++++++++++++++++++ 4 files changed, 245 insertions(+) create mode 100644 postgresql/dev_postgres_database/public/functions/udf_adjust_sequence.sql create mode 100644 postgresql/dev_postgres_database/public/functions/udf_check_value.sql create mode 100644 postgresql/dev_postgres_database/public/functions/udf_kill_connections_pg3.sql create mode 100644 postgresql/dev_postgres_database/public/functions/udf_kill_connections_pg4.sql diff --git a/postgresql/dev_postgres_database/public/functions/udf_adjust_sequence.sql b/postgresql/dev_postgres_database/public/functions/udf_adjust_sequence.sql new file mode 100644 index 0000000..ed29c85 --- /dev/null +++ b/postgresql/dev_postgres_database/public/functions/udf_adjust_sequence.sql @@ -0,0 +1,60 @@ +-- DROP FUNCTION udf_adjust_sequence); +CREATE OR REPLACE FUNCTION udf_adjust_sequence() +RETURNS INT AS +$BODY$ +DECLARE + d_seq_list record; + +BEGIN + + -- If the column value is greater than the sequence + FOR d_seq_list IN ( + + WITH tmp_sequence_list AS ( + + SELECT + c.table_schema || '.'|| c.table_name table_name, + c.column_name, + substring(c.column_default FROM position('(' in c.column_default) + 2 FOR (position('::' in c.column_default) - (position('(' in c.column_default) + 3)) ) column_default + FROM + information_schema.columns c + WHERE + c.data_type = 'integer' + AND table_schema NOT IN ('pg_catalog', 'pg_toast','pgagent','information_schema') + AND c.column_default ~~ 'nextval%' + AND c.column_default ~~ '%_seq%' + ORDER BY c.table_schema, c.table_name + + ), tmp_values AS ( + + SELECT + column_default, + ( + SELECT + udf_check_value('SELECT last_value FROM '|| column_default, TRUE)) sequence_value, + coalesce((SELECT udf_check_value('SELECT max(' || column_name || ') FROM '|| table_name, TRUE)), 0) column_value + FROM tmp_sequence_list + ) + SELECT * + FROM + tmp_values + WHERE + (column_value > sequence_value) or (sequence_value > (column_value+1)) + ) + LOOP + + PERFORM udf_check_value('ALTER SEQUENCE ' || d_seq_list.column_default || ' RESTART WITH ' || (d_seq_list.column_value + 1), FALSE); + + END LOOP; + + RETURN 1; + +END; + +$BODY$ +LANGUAGE plpgsql VOLATILE +COST 100; + +ALTER FUNCTION udf_adjust_sequence() + OWNER TO postgres; +COMMENT ON FUNCTION udf_adjust_sequence() IS 'Adjust all the Sequence which name ends with "_seq" in all the tables in all the schemas (excluding postgres)'; diff --git a/postgresql/dev_postgres_database/public/functions/udf_check_value.sql b/postgresql/dev_postgres_database/public/functions/udf_check_value.sql new file mode 100644 index 0000000..402ae85 --- /dev/null +++ b/postgresql/dev_postgres_database/public/functions/udf_check_value.sql @@ -0,0 +1,33 @@ +-- DROP FUNCTION udf_check_value(TEXT, BOOLEAN); +CREATE OR REPLACE FUNCTION udf_check_value( + i_query TEXT, + i_result BOOLEAN + ) +RETURNS INT AS +$BODY$ +DECLARE + d_value INT := 0; +BEGIN + /* + To test: + SELECT udf_check_value('SELECT last_value FROM seqname_id_seq', TRUE); + */ + + IF i_result THEN + EXECUTE i_query INTO d_value; + ELSE + EXECUTE i_query; + END IF; + + RETURN d_value; + +END; +$BODY$ + LANGUAGE plpgsql VOLATILE + COST 100; +ALTER FUNCTION udf_check_value(TEXT, BOOLEAN) + OWNER TO postgres; +COMMENT ON FUNCTION udf_check_value(TEXT, BOOLEAN) IS 'Returns the last value of a sequence'; + + + diff --git a/postgresql/dev_postgres_database/public/functions/udf_kill_connections_pg3.sql b/postgresql/dev_postgres_database/public/functions/udf_kill_connections_pg3.sql new file mode 100644 index 0000000..e587901 --- /dev/null +++ b/postgresql/dev_postgres_database/public/functions/udf_kill_connections_pg3.sql @@ -0,0 +1,76 @@ +-- DROP FUNCTION udf_kill_connections_pg3(INTEGER); +CREATE OR REPLACE FUNCTION udf_kill_connections_pg3(param_max_allowed_connections INTEGER) + RETURNS TEXT AS +$BODY$ + + DECLARE + +/* + + Author: Manuel Carrero and Simon Cedeno + Creation Date: 2018 + + To test: + SELECT udf_kill_connections_pg3(2); + +*/ + + i RECORD; + local_connections_killed INTEGER; + local_returning_message VARCHAR; + + BEGIN + + local_connections_killed = 0; + local_returning_message = ''; + + FOR i IN ( + SELECT + psa.pid + FROM + pg_stat_activity psa, + ( + SELECT + application_name, + client_addr, + COUNT(client_addr) AS quant + FROM + pg_stat_activity + GROUP BY + application_name, + client_addr + HAVING + COUNT(client_addr) > param_max_allowed_connections + ) AS connections + WHERE + psa.application_name IN ('pgAdmin III - Query Tool') + AND psa.client_addr = connections.client_addr + ) + LOOP + + PERFORM (SELECT pg_terminate_backend(i.pid)); + local_connections_killed = local_connections_killed + 1; + + END LOOP; + + IF (local_connections_killed = 0) THEN + + local_returning_message = 'Not found connections to Kill.'; + + RETURN local_returning_message; + + ELSE + + local_returning_message = 'Connections Killed, you probably will not see this message...'; + + RETURN local_returning_message; + + END IF; + + END; + +$BODY$ +LANGUAGE plpgsql VOLATILE +COST 100; + +COMMENT ON FUNCTION udf_kill_connections_pg3(INTEGER) IS 'This function is used to kill a determinate number of pgAdmin III connections, to prevent excessive windows opened.'; diff --git a/postgresql/dev_postgres_database/public/functions/udf_kill_connections_pg4.sql b/postgresql/dev_postgres_database/public/functions/udf_kill_connections_pg4.sql new file mode 100644 index 0000000..c3d4206 --- /dev/null +++ b/postgresql/dev_postgres_database/public/functions/udf_kill_connections_pg4.sql @@ -0,0 +1,76 @@ +-- DROP FUNCTION udf_kill_connections_pg4(INTEGER); +CREATE OR REPLACE FUNCTION udf_kill_connections_pg4(param_max_allowed_connections INTEGER) + RETURNS TEXT AS +$BODY$ + + DECLARE + +/* + + Author: Manuel Carrero and Simon Cedeno + Creation Date: 2018 + + To test: + SELECT udf_kill_connections_pg4(5); + +*/ + + i RECORD; + local_connections_killed INTEGER; + local_returning_message VARCHAR; + + BEGIN + + local_connections_killed = 0; + local_returning_message = ''; + + FOR i IN ( + SELECT + psa.pid + FROM + pg_stat_activity psa, + ( + SELECT + application_name, + client_addr, + COUNT(client_addr) AS quant + FROM + pg_stat_activity + GROUP BY + application_name, + client_addr + HAVING + COUNT(client_addr) > param_max_allowed_connections + ) AS connections + WHERE + psa.application_name LIKE 'pgAdmin 4 - CONN:%' + AND psa.client_addr = connections.client_addr + ) + LOOP + + PERFORM (SELECT pg_terminate_backend(i.pid)); + local_connections_killed = local_connections_killed + 1; + + END LOOP; + + IF (local_connections_killed = 0) THEN + + local_returning_message = 'Not found connections to Kill.'; + + RETURN local_returning_message; + + ELSE + + local_returning_message = 'Connections Killed, you probably will not see this message...'; + + RETURN local_returning_message; + + END IF; + + END; + +$BODY$ +LANGUAGE plpgsql VOLATILE +COST 100; + +COMMENT ON FUNCTION udf_kill_connections_pg4(INTEGER) IS 'This function is used to kill a determinate number of pgAdmin 4 connections, to prevent excessive windows opened.'; From 32f43088ab067070acb9dc855b63bf1dd58753af Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 6 Jul 2019 23:04:48 -0400 Subject: [PATCH 03/24] Adding etl examples folder: etl_migracion_general --- .../etl_examples/etl_migracion_general().sql | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 postgresql/etl_examples/etl_migracion_general().sql diff --git a/postgresql/etl_examples/etl_migracion_general().sql b/postgresql/etl_examples/etl_migracion_general().sql new file mode 100755 index 0000000..6887be5 --- /dev/null +++ b/postgresql/etl_examples/etl_migracion_general().sql @@ -0,0 +1,44 @@ +CREATE OR REPLACE FUNCTION etl_migracion_general() + RETURNS character varying AS +$BODY$ + DECLARE + + /* Migración de Datos usada para trasladar los datos del INAC (2017) + + Para probarlo: select etl_migracion_general() + + - Para eliminar datos de lotus: + + delete from lotus_usr_exist_datos_basicos; + delete from lotus_usr_exist_datos_exp_cert; + delete from lotus_usr_exist_datos_exp_hab; + delete from lotus_usr_exist_datos_exp_lic; + delete from lotus_usr_exist_descalif_med; + + */ + + -- Para valores de retorno + resultado_etl_datos_basicos varchar; + resultado_etl_exp_cert varchar; + resultado_etl_descalificaciones varchar; + resultado_etl_licencia varchar; + resultado_etl_habilitaciones varchar; + + valor varchar; + + BEGIN + + resultado_etl_datos_basicos = (select etl_datos_basicos()); + resultado_etl_exp_cert = (select etl_exp_cert()); + resultado_etl_descalificaciones = (select etl_descalificacion_medica()); + resultado_etl_licencia = (select etl_licencia()); + resultado_etl_habilitaciones = (select etl_habilitacion()); + + valor = 'Migración de datos finalizada'; + + RETURN valor; + +END; +$BODY$ + LANGUAGE plpgsql VOLATILE + COST 100; From 9ab7333f1ae9c68cdc29273c1d57c517c73f9e32 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 6 Jul 2019 23:06:05 -0400 Subject: [PATCH 04/24] Adding etl examples folder: etl_datos_basicos (solicitante) --- postgresql/etl_examples/etl_datos_basicos.sql | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100755 postgresql/etl_examples/etl_datos_basicos.sql diff --git a/postgresql/etl_examples/etl_datos_basicos.sql b/postgresql/etl_examples/etl_datos_basicos.sql new file mode 100755 index 0000000..20635e4 --- /dev/null +++ b/postgresql/etl_examples/etl_datos_basicos.sql @@ -0,0 +1,130 @@ +CREATE OR REPLACE FUNCTION etl_datos_basicos() + RETURNS character varying AS +$BODY$ + DECLARE + + /* + Resumen: destinado a migrar los datos de la bd de lotus a la bd de limed. + Se encarga de extraer los datos de la tabla lotus_usr_exist_datos_basicos y los inserta en las tablas + tbl_solicitante y tbl_dni + */ + + nacionalidad varchar(100); + tipo_documento2 varchar(100); + id_solicitante2 int; + solicitante_repetido boolean; + + -- Extraccion de Datos Basicos -- + consulta cursor for select * from lotus_usr_exist_datos_basicos; + arreglo lotus_usr_exist_datos_basicos%ROWTYPE; + + valor varchar; + + BEGIN + + FOR arreglo IN consulta LOOP + + if not exists(select id_solicitante from tbl_solicitante where id_solicitante = arreglo.usr_exist_datos_bas_id) then + + nacionalidad := arreglo.usr_exist_datos_bas_nacionalidad; + + -- Transformacion y Carga de Datos -- + if (nacionalidad= 'V') then + + tipo_documento2:= 'Cédula'; + + elsif (nacionalidad= 'P') then + + tipo_documento2:= 'Pasaporte'; + + elsif (nacionalidad = 'D') then + + tipo_documento2:= 'Dni'; + + elsif (nacionalidad = 'R') then + + tipo_documento2:= 'RIF'; + + elsif (nacionalidad = 'E') then + + tipo_documento2:= 'Extranjero'; + + end if; + + if (nacionalidad = 'V') then + + -- Cargado de Datos de en tbl_solicitante + INSERT INTO tbl_solicitante ( + id_solicitante, + cod_solicitante, + tx_nombre, + tx_apellido, + -- fec_nacimiento, + id_nacionalidad, + id_pais + ) + VALUES ( + arreglo.usr_exist_datos_bas_id, + cast(arreglo.usr_exist_datos_bas_cod_clte as int), + arreglo.usr_exist_datos_bas_nombre, + arreglo.usr_exist_datos_bas_apellido, + -- arreglo.usr_exist_datos_bas_fec_nac, + 196, + 238 + ); + + else + -- Cargado de Datos de en tbl_solicitante + INSERT INTO tbl_solicitante ( + id_solicitante, + cod_usuario, + tx_nombre, + tx_apellido, + -- fec_nacimiento, + id_nacionalidad, + id_pais + ) + VALUES ( + arreglo.usr_exist_datos_bas_id, + cast(arreglo.usr_exist_datos_bas_cod_clte as int), + arreglo.usr_exist_datos_bas_nombre, + arreglo.usr_exist_datos_bas_apellido, + -- arreglo.usr_exist_datos_bas_fec_nac, + 201, + 250 + ); + + end if; + + -- Carga de Datos en tbl_dni + INSERT INTO tbl_dni ( + id_tbl_dni, + tipo_documento, + num_documento, + id_solicitante + ) + VALUES ( + arreglo.usr_exist_datos_bas_id, + tipo_documento2, + upper(arreglo.usr_exist_datos_bas_cedula_pasaporte), + arreglo.usr_exist_datos_bas_id + ); + + raise notice 'Insertado el solicitante %',arreglo.usr_exist_datos_bas_id; + + else + + raise notice 'El Solicitante % % N°identidad % con el id: % ya se encuentra en la bd',arreglo.usr_exist_datos_bas_nombre, arreglo.usr_exist_datos_bas_apellido,arreglo.usr_exist_datos_bas_cedula_pasaporte,arreglo.usr_exist_datos_bas_id; + + end if; + + END LOOP; + + valor = 'Migración de datos basicos finalizada'; + + RETURN valor; + +END; +$BODY$ + LANGUAGE plpgsql VOLATILE + COST 100; From 51370d6676f4cd01530c130032f658ecfff34517 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 6 Jul 2019 23:07:29 -0400 Subject: [PATCH 05/24] Adding etl examples folder: etl_exp_cert (certificados medicos) --- postgresql/etl_examples/etl_exp_cert.sql | 80 ++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 postgresql/etl_examples/etl_exp_cert.sql diff --git a/postgresql/etl_examples/etl_exp_cert.sql b/postgresql/etl_examples/etl_exp_cert.sql new file mode 100755 index 0000000..2e33969 --- /dev/null +++ b/postgresql/etl_examples/etl_exp_cert.sql @@ -0,0 +1,80 @@ +CREATE OR REPLACE FUNCTION etl_exp_cert() + RETURNS character varying AS +$BODY$ + DECLARE + + /* + Resumen: destinado a migrar los datos de la bd de lotus a la bd de limed. + Se encarga de extraer los datos de la tabla lotus_usr_exist_datos_exp_cert y los inserta en la tabla tbl_documento + */ + + -- Extraccion de los datos de los Certificados de Lotus + consulta2 cursor for select * from lotus_usr_exist_datos_exp_cert order by usr_exist_datos_exp_cert_fec_venc desc; + arreglo lotus_usr_exist_datos_exp_cert%ROWTYPE; + + d_num_documento varchar; + d_fecha_expedicion date; + d_fecha_vencimiento date; + valor varchar; + + BEGIN + + FOR arreglo IN consulta2 LOOP + + if not exists(select id_documento from tbl_documento where id_documento = arreglo.usr_exist_datos_bas_id) then + + select num_documento into strict d_num_documento from tbl_dni where id_solicitante = arreglo.usr_exist_datos_bas_id; + + -- Transformacion y Carga de Datos -- + + -- Se acomodan las fechas incorrectas + if(arreglo.usr_exist_datos_exp_cert_fec_exp = '%0001-01-01 BC%') then + d_fecha_expedicion = '0001-01-01'; + elsif(arreglo.usr_exist_datos_exp_cert_fec_venc = '%0001-01-01 BC%') then + d_fecha_vencimiento = '0001-01-01'; + elsif(arreglo.usr_exist_datos_exp_cert_fec_exp is null) then + d_fecha_expedicion = '0001-01-01'; + elsif(arreglo.usr_exist_datos_exp_cert_fec_venc is null) then + d_fecha_vencimiento = '0001-01-01'; + else + d_fecha_expedicion = arreglo.usr_exist_datos_exp_cert_fec_exp; + d_fecha_vencimiento = arreglo.usr_exist_datos_exp_cert_fec_venc; + end if; + + INSERT INTO tbl_documento ( + id_documento, + solicitante, + num_documento2, + clase, + fec_emic, + fec_vcto, + tipo_documento + ) + VALUES ( + arreglo.usr_exist_datos_exp_cert_id, + arreglo.usr_exist_datos_bas_id, + upper(d_num_documento), + arreglo.clase_cert_med_id, + d_fecha_expedicion, + d_fecha_vencimiento, + 2 + ); + + raise notice 'Insertando el Documento: %',arreglo.usr_exist_datos_exp_cert_id; + + else + + raise notice 'El Documento con id % ya se encuentra en la bd',arreglo.usr_exist_datos_exp_cert_id; + + end if; + + END LOOP; + + valor = 'Migración de expedientes de certificados finalizada'; + + RETURN valor; + +END; +$BODY$ + LANGUAGE plpgsql VOLATILE + COST 100; \ No newline at end of file From e51de25e8b1d0535189a8fc0c16ff9d95e64c2d2 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 6 Jul 2019 23:08:22 -0400 Subject: [PATCH 06/24] Adding etl examples folder: etl_descalificacion_medica (descalificaciones medicas de los solicitantes) --- .../etl_descalificacion_medica().sql | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 postgresql/etl_examples/etl_descalificacion_medica().sql diff --git a/postgresql/etl_examples/etl_descalificacion_medica().sql b/postgresql/etl_examples/etl_descalificacion_medica().sql new file mode 100755 index 0000000..45cc144 --- /dev/null +++ b/postgresql/etl_examples/etl_descalificacion_medica().sql @@ -0,0 +1,78 @@ +CREATE OR REPLACE FUNCTION etl_descalificacion_medica() + RETURNS character varying AS +$BODY$ + DECLARE + + + /* + Resumen: destinado a migrar los datos de la bd de lotus a la bd de limed. + Se encarga de extraer los datos de la tabla lotus_usr_exist_descalif_med y los inserta en la tabla tbl_descalificacion_medica + ademas, actualiza el estatus_solicitante en la tabla tbl_documento asociados a todos los documentos correspondientes + al respectivo solicitante. + */ + + -- Extraccion de Datos de Descalificaciones Medicas-- + consulta5 cursor for select * from lotus_usr_exist_descalif_med; + arreglo lotus_usr_exist_descalif_med%ROWTYPE; + + arreglo_doc record; + + valor varchar; + + BEGIN + + FOR arreglo IN consulta5 LOOP + + -- Se verifica si ya existe el registro en la base de datos + if not exists(select id_descalificacion_lotus from tbl_descalificacion_medica where id_descalificacion_lotus = arreglo.usr_exist_descalif_med_id) then + + -- Se aplican las descalificaciones medicas a los correspondientes certificados (cap o cma) asociados + if (arreglo.tipo_descalif_med_id = 1) then + + update tbl_documento set descalificacion_medica = true, status_solicitante = 21, tx_motivo = arreglo.usr_exist_descalif_med_motivo where solicitante = arreglo.usr_exist_datos_bas_id; + + elsif (arreglo.tipo_descalif_med_id = 2) then + + update tbl_documento set descalificacion_medica = true, status_solicitante = 20, tx_motivo = arreglo.usr_exist_descalif_med_motivo where solicitante = arreglo.usr_exist_datos_bas_id; + + end if; + + FOR arreglo_doc IN select id_documento,solicitante from tbl_documento where solicitante = arreglo.usr_exist_datos_bas_id LOOP + + if (arreglo_doc.solicitante = arreglo.usr_exist_datos_bas_id) then + -- Transformacion y Carga de Datos -- + INSERT INTO tbl_descalificacion_medica ( + id_descalificacion_lotus, + documento, + tipo_descalif_medica, + motivo + ) + VALUES ( + arreglo.usr_exist_descalif_med_id, + arreglo_doc.id_documento, + arreglo.tipo_descalif_med_id, + arreglo.usr_exist_descalif_med_motivo + ); + + raise notice 'Inserte la descalificacion %, el documento fue % y el solicitante en desc fue %',arreglo.usr_exist_descalif_med_id,arreglo_doc.id_documento,arreglo.usr_exist_datos_bas_id; + + end if; + + end loop; + + else + + raise notice 'La descalificacion con id % ya se encuentra en la bd',arreglo.usr_exist_descalif_med_id; + + end if; + + valor = 'Migración de descalificaciones medicas finalizada'; + + END LOOP; + + RETURN valor; + +END; +$BODY$ + LANGUAGE plpgsql VOLATILE + COST 100; From bc54e46c0b9574ee9c5a02b33678428d0d4cc6f7 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 6 Jul 2019 23:08:59 -0400 Subject: [PATCH 07/24] Adding etl examples folder: etl_licencia (licencias de solicitantes) --- postgresql/etl_examples/etl_licencia.sql | 81 ++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 postgresql/etl_examples/etl_licencia.sql diff --git a/postgresql/etl_examples/etl_licencia.sql b/postgresql/etl_examples/etl_licencia.sql new file mode 100755 index 0000000..83e401a --- /dev/null +++ b/postgresql/etl_examples/etl_licencia.sql @@ -0,0 +1,81 @@ +CREATE OR REPLACE FUNCTION etl_licencia() + RETURNS character varying AS +$BODY$ + DECLARE + + /* + Resumen: destinado a migrar los datos de la bd de lotus a la bd de limed. + Se encarga de extraer los datos de la tabla lotus_usr_exist_datos_exp_lic y los inserta en la tabla tbl_licencia + */ + + d_status2 int; + numero_licencia2 character varying(50); + d_fecha_expedicion date; + + -- Extraccion de Datos -- + consulta3 cursor for select * from lotus_usr_exist_datos_exp_lic; + arreglo lotus_usr_exist_datos_exp_lic%ROWTYPE; + + valor varchar; + + BEGIN + + FOR arreglo IN consulta3 LOOP + + if not exists(select id_licencia from tbl_licencia where id_licencia = arreglo.usr_exist_datos_exp_lic_id) then + + numero_licencia2 = arreglo.usr_exist_datos_exp_lic_numero; + + -- Transformacion y Carga de Datos -- + + -- Se homologan los 3 estatus de la bd lotus, haciendo correspondencia con los estatus cargados en la bd limed + if(arreglo.usr_exist_datos_exp_lic_estatus = '0') then -- No especificado -- + d_status2 = 0; + elsif(arreglo.usr_exist_datos_exp_lic_estatus = '1') then -- Vigente -- + d_status2 = 1; + elsif(arreglo.usr_exist_datos_exp_lic_estatus = '2') then -- Vencida -- + d_status2 = 2; + end if; + + -- Se acomodan las fechas incorrectas + if(arreglo.usr_exist_datos_exp_lic_fec_emi = '%0001-01-01 BC%') then + d_fecha_expedicion = '0001-01-01'; + elsif(arreglo.usr_exist_datos_exp_lic_fec_emi is null) then + d_fecha_expedicion = '0001-01-01'; + else + d_fecha_expedicion = arreglo.usr_exist_datos_exp_lic_fec_emi; + end if; + + INSERT INTO tbl_licencia ( + id_licencia, + id_solicitante, + fec_emision, + tipo_licencia, + status, + numero_licencia2 + ) + VALUES ( + arreglo.usr_exist_datos_exp_lic_id, + arreglo.usr_exist_datos_bas_id, + d_fecha_expedicion, + arreglo.tipo_pers_id, + d_status2, + upper(numero_licencia2) + ); + + else + + raise notice 'La licencia con id % ya se encuentra en la bd',arreglo.usr_exist_datos_exp_lic_id; + + end if; + + valor = 'Migración de licencias finalizada'; + + END LOOP; + + RETURN valor; + +END; +$BODY$ + LANGUAGE plpgsql VOLATILE + COST 100; From c69ec0fbc4031f6ff29b476907d2045c42860475 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 6 Jul 2019 23:09:46 -0400 Subject: [PATCH 08/24] Adding etl examples folder: etl_habilitacion (habilitaciones de solicitantes) --- .../etl_examples/etl_habilitacion().sql | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100755 postgresql/etl_examples/etl_habilitacion().sql diff --git a/postgresql/etl_examples/etl_habilitacion().sql b/postgresql/etl_examples/etl_habilitacion().sql new file mode 100755 index 0000000..d9a822c --- /dev/null +++ b/postgresql/etl_examples/etl_habilitacion().sql @@ -0,0 +1,146 @@ +CREATE OR REPLACE FUNCTION etl_habilitacion() + RETURNS character varying AS +$BODY$ + DECLARE + + + /* + Resumen: destinado a migrar los datos de la bd de lotus a la bd de limed. + Se encarga de extraer los datos de la tabla lotus_usr_exist_datos_exp_hab y los inserta en la tabla tbl_habilitacion + */ + + d_funcion integer; +-- d_clase integer; +-- d_tipo integer; + d_verificar_tipo boolean; + d_verificar_clase boolean; + d_fecha_expedicion date; + d_fecha_vencimiento date; + + -- Extraccion de Datos -- + consulta4 cursor for select * from lotus_usr_exist_datos_exp_hab; + arreglo lotus_usr_exist_datos_exp_hab%ROWTYPE; + + i record; + + valor varchar; + + BEGIN + + FOR arreglo IN consulta4 LOOP + + if not exists(select id_habilitacion from tbl_habilitacion where id_habilitacion = arreglo.usr_exist_datos_exp_hab_id) then + for i in select usr_exist_datos_exp_lic_id from lotus_usr_exist_datos_exp_lic where usr_exist_datos_exp_lic_id = arreglo.usr_exist_datos_exp_lic_id loop + + -- Se verifica si la habilitacion es de clase + if (arreglo.tipo_habilitacion_id = 1) then + d_verificar_clase = true; + d_verificar_tipo = false; + + -- Se verifica si la habilitacion es de tipo + else + d_verificar_tipo = true; + d_verificar_clase = false; + + end if; + + -- Transformacion y Carga de Datos -- + + -- Se acomodan las fechas incorrectas + if(arreglo.usr_exist_datos_exp_hab_fec_exp = '%0001-01-01 BC%') then + d_fecha_expedicion = '0001-01-01'; + elsif(arreglo.usr_exist_datos_exp_hab_fec_venc = '%0001-01-01 BC%') then + d_fecha_vencimiento = '0001-01-01'; + elsif(arreglo.usr_exist_datos_exp_hab_fec_exp is null) then + d_fecha_expedicion = '0001-01-01'; + elsif(arreglo.usr_exist_datos_exp_hab_fec_venc is null) then + d_fecha_vencimiento = '0001-01-01'; + else + d_fecha_expedicion = arreglo.usr_exist_datos_exp_hab_fec_exp; + d_fecha_vencimiento = arreglo.usr_exist_datos_exp_hab_fec_venc; + end if; + + if (arreglo.tipo_habilitacion_id = 1) then + + INSERT INTO tbl_habilitacion ( + id_habilitacion, + clase, + tipo, + desc_habilitacion, + fec_habilitacion, + fec_vencimiento, + funcion, + tipo_licencia, + categoria, + verificar_clase, + verificar_tipo + ) + VALUES ( + arreglo.usr_exist_datos_exp_hab_id, + 1000000, + 1000000, + 'No Especificado', + d_fecha_expedicion, + d_fecha_vencimiento, + d_funcion, + arreglo.usr_exist_datos_exp_lic_id, + 100, + d_verificar_clase, + d_verificar_tipo + ); + + raise notice 'Inserte la habilitacion % y es de Clase',arreglo.usr_exist_datos_exp_hab_id; + + elsif (arreglo.tipo_habilitacion_id = 2) then + + -- Transformacion y Carga de Datos -- + INSERT INTO tbl_habilitacion ( + id_habilitacion, + clase, + tipo, + desc_habilitacion, + fec_habilitacion, + fec_vencimiento, + funcion, + tipo_licencia, + categoria, + verificar_clase, + verificar_tipo + ) + VALUES ( + arreglo.usr_exist_datos_exp_hab_id, + 1000000, + 1000000, + 'No Especificado', + d_fecha_expedicion, + d_fecha_vencimiento, + d_funcion, + arreglo.usr_exist_datos_exp_lic_id, + 100, + d_verificar_clase, + d_verificar_tipo + ); + + raise notice 'Inserte la habilitacion % y es de Tipo',arreglo.usr_exist_datos_exp_hab_id; + + end if; + + END LOOP; + + else + + raise notice 'La habilitacion con id % ya se encuentra en la bd',arreglo.usr_exist_datos_exp_hab_id; + + end if; + + + END LOOP; + + valor = 'Migración de habilitaciones finalizada'; + + RETURN valor; + +END; +$BODY$ + LANGUAGE plpgsql VOLATILE + COST 100; From efe4e4aee84078e70c07f63dc0e55ec4ed57c918 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 6 Jul 2019 23:32:08 -0400 Subject: [PATCH 09/24] Updating readme and some stuffs --- README.md | 30 ++++++++++++++----- execute_all.sh | 3 -- postgresql/etl_examples/etl_datos_basicos.sql | 2 +- .../etl_descalificacion_medica().sql | 2 +- postgresql/etl_examples/etl_exp_cert.sql | 2 +- .../etl_examples/etl_habilitacion().sql | 2 +- postgresql/etl_examples/etl_licencia.sql | 2 +- .../etl_examples/etl_migracion_general().sql | 2 +- 8 files changed, 29 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 8ca565d..8e1d305 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,33 @@ -This a repository for scripts and sql querys +# postgresql-useful-scripts + +This a repository for useful PostgreSQL scripts and sql querys ------------------------ -1) postgresql +## Use + +Give permissions to execute_all.sh file: + + chmod +x execute_all.sh + +Login as a postgres user: + + sudo -i -u postgres + +And execute the execute_all.sh file: -* execute_all.sh: master file to execute all the sql files. + ./execute_all.sh + +## Structure: Files are contained inside of the dev_postgres_database folder, with the below structure: -a) schema -> useful_queries: a several useful querys related with structures of postgres (schemas, sequences, tables, functions, triggers), dba monitoring querys and more. +a) schema -> useful_queries: + +- dba_scripts.sql: some dba monitoring querys. + +- useful_structure_queries.sql: several useful querys related with schemas, sequences, tables, columns, triggers, constraints, functions and channels. -b) schema -> functions: a several PL/pgSQL functions with useful code. +b) schema -> functions: several PL/pgSQL functions with useful code (e.g: generating queries with tree json format, killing connections in pgadmin and more). c) schema -> tables: examples tables to use in the plpgsql_scripts. @@ -20,6 +38,4 @@ Contributions All work to improve performance is good - - Enjoy it! diff --git a/execute_all.sh b/execute_all.sh index c7343bb..74fc181 100755 --- a/execute_all.sh +++ b/execute_all.sh @@ -1,6 +1,3 @@ -# Execute from postgres user: -# chmod +x execute_all.sh -# sudo -i -u postgres psql -U postgres -a -f ./postgresql/main_database.sql psql -U postgres -d dev_postgres_database -a -c "CREATE SCHEMA temporal_schema;" #psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udt_general_tables_insert.sql diff --git a/postgresql/etl_examples/etl_datos_basicos.sql b/postgresql/etl_examples/etl_datos_basicos.sql index 20635e4..63673d3 100755 --- a/postgresql/etl_examples/etl_datos_basicos.sql +++ b/postgresql/etl_examples/etl_datos_basicos.sql @@ -4,7 +4,7 @@ $BODY$ DECLARE /* - Resumen: destinado a migrar los datos de la bd de lotus a la bd de limed. + Resumen: destinado a migrar los datos de la bd de origen a la nueva bd. Se encarga de extraer los datos de la tabla lotus_usr_exist_datos_basicos y los inserta en las tablas tbl_solicitante y tbl_dni */ diff --git a/postgresql/etl_examples/etl_descalificacion_medica().sql b/postgresql/etl_examples/etl_descalificacion_medica().sql index 45cc144..2415e83 100755 --- a/postgresql/etl_examples/etl_descalificacion_medica().sql +++ b/postgresql/etl_examples/etl_descalificacion_medica().sql @@ -5,7 +5,7 @@ $BODY$ /* - Resumen: destinado a migrar los datos de la bd de lotus a la bd de limed. + Resumen: destinado a migrar los datos de la bd de origen a la nueva bd. Se encarga de extraer los datos de la tabla lotus_usr_exist_descalif_med y los inserta en la tabla tbl_descalificacion_medica ademas, actualiza el estatus_solicitante en la tabla tbl_documento asociados a todos los documentos correspondientes al respectivo solicitante. diff --git a/postgresql/etl_examples/etl_exp_cert.sql b/postgresql/etl_examples/etl_exp_cert.sql index 2e33969..69e1e38 100755 --- a/postgresql/etl_examples/etl_exp_cert.sql +++ b/postgresql/etl_examples/etl_exp_cert.sql @@ -4,7 +4,7 @@ $BODY$ DECLARE /* - Resumen: destinado a migrar los datos de la bd de lotus a la bd de limed. + Resumen: destinado a migrar los datos de la bd de origen a la nueva bd. Se encarga de extraer los datos de la tabla lotus_usr_exist_datos_exp_cert y los inserta en la tabla tbl_documento */ diff --git a/postgresql/etl_examples/etl_habilitacion().sql b/postgresql/etl_examples/etl_habilitacion().sql index d9a822c..87a3222 100755 --- a/postgresql/etl_examples/etl_habilitacion().sql +++ b/postgresql/etl_examples/etl_habilitacion().sql @@ -5,7 +5,7 @@ $BODY$ /* - Resumen: destinado a migrar los datos de la bd de lotus a la bd de limed. + Resumen: destinado a migrar los datos de la bd de origen a la nueva bd. Se encarga de extraer los datos de la tabla lotus_usr_exist_datos_exp_hab y los inserta en la tabla tbl_habilitacion */ diff --git a/postgresql/etl_examples/etl_licencia.sql b/postgresql/etl_examples/etl_licencia.sql index 83e401a..3f5bbc5 100755 --- a/postgresql/etl_examples/etl_licencia.sql +++ b/postgresql/etl_examples/etl_licencia.sql @@ -4,7 +4,7 @@ $BODY$ DECLARE /* - Resumen: destinado a migrar los datos de la bd de lotus a la bd de limed. + Resumen: destinado a migrar los datos de la bd de origen a la nueva bd. Se encarga de extraer los datos de la tabla lotus_usr_exist_datos_exp_lic y los inserta en la tabla tbl_licencia */ diff --git a/postgresql/etl_examples/etl_migracion_general().sql b/postgresql/etl_examples/etl_migracion_general().sql index 6887be5..cb0749a 100755 --- a/postgresql/etl_examples/etl_migracion_general().sql +++ b/postgresql/etl_examples/etl_migracion_general().sql @@ -3,7 +3,7 @@ CREATE OR REPLACE FUNCTION etl_migracion_general() $BODY$ DECLARE - /* Migración de Datos usada para trasladar los datos del INAC (2017) + /* Migración de Datos usada para trasladar los datos de un proyecto del sector aeronáutico Para probarlo: select etl_migracion_general() From 40d8167a20fc447caf952637bb488b6aee5ac8bb Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 6 Jul 2019 23:48:06 -0400 Subject: [PATCH 10/24] Adding etl exampls information --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8e1d305..d6cdbcf 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # postgresql-useful-scripts This a repository for useful PostgreSQL scripts and sql querys ------------------------- ## Use @@ -19,20 +18,26 @@ And execute the execute_all.sh file: ## Structure: +* dev_postgres_database folder: + Files are contained inside of the dev_postgres_database folder, with the below structure: a) schema -> useful_queries: - dba_scripts.sql: some dba monitoring querys. -- useful_structure_queries.sql: several useful querys related with schemas, sequences, tables, columns, triggers, constraints, functions and channels. +- useful_structure_queries.sql: several useful querys related with schemas, sequences, tables, columns, triggers, constraints, functions and pg_notify channels. -b) schema -> functions: several PL/pgSQL functions with useful code (e.g: generating queries with tree json format, killing connections in pgadmin and more). +b) schema -> functions: several PL/PL/PgSQL functions with useful code (e.g: generating queries with tree json format, killing connections in pgadmin and more). c) schema -> tables: examples tables to use in the plpgsql_scripts. d) schema -> data: mock data to use in the plpgsql_scripts. +* etl_examples folder: + +Some etl examples developed in PL/PgSQL, with useful code (e.g cursors, data cleaning and more) + Contributions ----------------------- From 5e49530d84a620112289ac46cf4d610741e44c4c Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Tue, 9 Jul 2019 22:15:26 -0400 Subject: [PATCH 11/24] Refactoring repository, moving several scripts to https://github.com/LegolasVzla/pgplsql-useful-examples --- .../functions}/dba_scripts.sql | 0 .../functions/udt_general_tables_insert.sql | 0 .../functions}/useful_structure_queries.sql | 0 .../functions/udf_kill_connections_pg3.sql | 76 --------- .../functions/udf_kill_connections_pg4.sql | 76 --------- postgresql/etl_examples/etl_datos_basicos.sql | 130 ---------------- .../etl_descalificacion_medica().sql | 78 ---------- postgresql/etl_examples/etl_exp_cert.sql | 80 ---------- .../etl_examples/etl_habilitacion().sql | 146 ------------------ postgresql/etl_examples/etl_licencia.sql | 81 ---------- .../etl_examples/etl_migracion_general().sql | 44 ------ 11 files changed, 711 deletions(-) rename postgresql/dev_postgres_database/{temporal_schema/useful_queries => public/functions}/dba_scripts.sql (100%) rename postgresql/dev_postgres_database/{temporal_schema => public}/functions/udt_general_tables_insert.sql (100%) rename postgresql/dev_postgres_database/{temporal_schema/useful_queries => public/functions}/useful_structure_queries.sql (100%) delete mode 100644 postgresql/dev_postgres_database/temporal_schema/functions/udf_kill_connections_pg3.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/functions/udf_kill_connections_pg4.sql delete mode 100755 postgresql/etl_examples/etl_datos_basicos.sql delete mode 100755 postgresql/etl_examples/etl_descalificacion_medica().sql delete mode 100755 postgresql/etl_examples/etl_exp_cert.sql delete mode 100755 postgresql/etl_examples/etl_habilitacion().sql delete mode 100755 postgresql/etl_examples/etl_licencia.sql delete mode 100755 postgresql/etl_examples/etl_migracion_general().sql diff --git a/postgresql/dev_postgres_database/temporal_schema/useful_queries/dba_scripts.sql b/postgresql/dev_postgres_database/public/functions/dba_scripts.sql similarity index 100% rename from postgresql/dev_postgres_database/temporal_schema/useful_queries/dba_scripts.sql rename to postgresql/dev_postgres_database/public/functions/dba_scripts.sql diff --git a/postgresql/dev_postgres_database/temporal_schema/functions/udt_general_tables_insert.sql b/postgresql/dev_postgres_database/public/functions/udt_general_tables_insert.sql similarity index 100% rename from postgresql/dev_postgres_database/temporal_schema/functions/udt_general_tables_insert.sql rename to postgresql/dev_postgres_database/public/functions/udt_general_tables_insert.sql diff --git a/postgresql/dev_postgres_database/temporal_schema/useful_queries/useful_structure_queries.sql b/postgresql/dev_postgres_database/public/functions/useful_structure_queries.sql similarity index 100% rename from postgresql/dev_postgres_database/temporal_schema/useful_queries/useful_structure_queries.sql rename to postgresql/dev_postgres_database/public/functions/useful_structure_queries.sql diff --git a/postgresql/dev_postgres_database/temporal_schema/functions/udf_kill_connections_pg3.sql b/postgresql/dev_postgres_database/temporal_schema/functions/udf_kill_connections_pg3.sql deleted file mode 100644 index e587901..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/functions/udf_kill_connections_pg3.sql +++ /dev/null @@ -1,76 +0,0 @@ --- DROP FUNCTION udf_kill_connections_pg3(INTEGER); -CREATE OR REPLACE FUNCTION udf_kill_connections_pg3(param_max_allowed_connections INTEGER) - RETURNS TEXT AS -$BODY$ - - DECLARE - -/* - - Author: Manuel Carrero and Simon Cedeno - Creation Date: 2018 - - To test: - SELECT udf_kill_connections_pg3(2); - -*/ - - i RECORD; - local_connections_killed INTEGER; - local_returning_message VARCHAR; - - BEGIN - - local_connections_killed = 0; - local_returning_message = ''; - - FOR i IN ( - SELECT - psa.pid - FROM - pg_stat_activity psa, - ( - SELECT - application_name, - client_addr, - COUNT(client_addr) AS quant - FROM - pg_stat_activity - GROUP BY - application_name, - client_addr - HAVING - COUNT(client_addr) > param_max_allowed_connections - ) AS connections - WHERE - psa.application_name IN ('pgAdmin III - Query Tool') - AND psa.client_addr = connections.client_addr - ) - LOOP - - PERFORM (SELECT pg_terminate_backend(i.pid)); - local_connections_killed = local_connections_killed + 1; - - END LOOP; - - IF (local_connections_killed = 0) THEN - - local_returning_message = 'Not found connections to Kill.'; - - RETURN local_returning_message; - - ELSE - - local_returning_message = 'Connections Killed, you probably will not see this message...'; - - RETURN local_returning_message; - - END IF; - - END; - -$BODY$ -LANGUAGE plpgsql VOLATILE -COST 100; - -COMMENT ON FUNCTION udf_kill_connections_pg3(INTEGER) IS 'This function is used to kill a determinate number of pgAdmin III connections, to prevent excessive windows opened.'; diff --git a/postgresql/dev_postgres_database/temporal_schema/functions/udf_kill_connections_pg4.sql b/postgresql/dev_postgres_database/temporal_schema/functions/udf_kill_connections_pg4.sql deleted file mode 100644 index c3d4206..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/functions/udf_kill_connections_pg4.sql +++ /dev/null @@ -1,76 +0,0 @@ --- DROP FUNCTION udf_kill_connections_pg4(INTEGER); -CREATE OR REPLACE FUNCTION udf_kill_connections_pg4(param_max_allowed_connections INTEGER) - RETURNS TEXT AS -$BODY$ - - DECLARE - -/* - - Author: Manuel Carrero and Simon Cedeno - Creation Date: 2018 - - To test: - SELECT udf_kill_connections_pg4(5); - -*/ - - i RECORD; - local_connections_killed INTEGER; - local_returning_message VARCHAR; - - BEGIN - - local_connections_killed = 0; - local_returning_message = ''; - - FOR i IN ( - SELECT - psa.pid - FROM - pg_stat_activity psa, - ( - SELECT - application_name, - client_addr, - COUNT(client_addr) AS quant - FROM - pg_stat_activity - GROUP BY - application_name, - client_addr - HAVING - COUNT(client_addr) > param_max_allowed_connections - ) AS connections - WHERE - psa.application_name LIKE 'pgAdmin 4 - CONN:%' - AND psa.client_addr = connections.client_addr - ) - LOOP - - PERFORM (SELECT pg_terminate_backend(i.pid)); - local_connections_killed = local_connections_killed + 1; - - END LOOP; - - IF (local_connections_killed = 0) THEN - - local_returning_message = 'Not found connections to Kill.'; - - RETURN local_returning_message; - - ELSE - - local_returning_message = 'Connections Killed, you probably will not see this message...'; - - RETURN local_returning_message; - - END IF; - - END; - -$BODY$ -LANGUAGE plpgsql VOLATILE -COST 100; - -COMMENT ON FUNCTION udf_kill_connections_pg4(INTEGER) IS 'This function is used to kill a determinate number of pgAdmin 4 connections, to prevent excessive windows opened.'; diff --git a/postgresql/etl_examples/etl_datos_basicos.sql b/postgresql/etl_examples/etl_datos_basicos.sql deleted file mode 100755 index 63673d3..0000000 --- a/postgresql/etl_examples/etl_datos_basicos.sql +++ /dev/null @@ -1,130 +0,0 @@ -CREATE OR REPLACE FUNCTION etl_datos_basicos() - RETURNS character varying AS -$BODY$ - DECLARE - - /* - Resumen: destinado a migrar los datos de la bd de origen a la nueva bd. - Se encarga de extraer los datos de la tabla lotus_usr_exist_datos_basicos y los inserta en las tablas - tbl_solicitante y tbl_dni - */ - - nacionalidad varchar(100); - tipo_documento2 varchar(100); - id_solicitante2 int; - solicitante_repetido boolean; - - -- Extraccion de Datos Basicos -- - consulta cursor for select * from lotus_usr_exist_datos_basicos; - arreglo lotus_usr_exist_datos_basicos%ROWTYPE; - - valor varchar; - - BEGIN - - FOR arreglo IN consulta LOOP - - if not exists(select id_solicitante from tbl_solicitante where id_solicitante = arreglo.usr_exist_datos_bas_id) then - - nacionalidad := arreglo.usr_exist_datos_bas_nacionalidad; - - -- Transformacion y Carga de Datos -- - if (nacionalidad= 'V') then - - tipo_documento2:= 'Cédula'; - - elsif (nacionalidad= 'P') then - - tipo_documento2:= 'Pasaporte'; - - elsif (nacionalidad = 'D') then - - tipo_documento2:= 'Dni'; - - elsif (nacionalidad = 'R') then - - tipo_documento2:= 'RIF'; - - elsif (nacionalidad = 'E') then - - tipo_documento2:= 'Extranjero'; - - end if; - - if (nacionalidad = 'V') then - - -- Cargado de Datos de en tbl_solicitante - INSERT INTO tbl_solicitante ( - id_solicitante, - cod_solicitante, - tx_nombre, - tx_apellido, - -- fec_nacimiento, - id_nacionalidad, - id_pais - ) - VALUES ( - arreglo.usr_exist_datos_bas_id, - cast(arreglo.usr_exist_datos_bas_cod_clte as int), - arreglo.usr_exist_datos_bas_nombre, - arreglo.usr_exist_datos_bas_apellido, - -- arreglo.usr_exist_datos_bas_fec_nac, - 196, - 238 - ); - - else - -- Cargado de Datos de en tbl_solicitante - INSERT INTO tbl_solicitante ( - id_solicitante, - cod_usuario, - tx_nombre, - tx_apellido, - -- fec_nacimiento, - id_nacionalidad, - id_pais - ) - VALUES ( - arreglo.usr_exist_datos_bas_id, - cast(arreglo.usr_exist_datos_bas_cod_clte as int), - arreglo.usr_exist_datos_bas_nombre, - arreglo.usr_exist_datos_bas_apellido, - -- arreglo.usr_exist_datos_bas_fec_nac, - 201, - 250 - ); - - end if; - - -- Carga de Datos en tbl_dni - INSERT INTO tbl_dni ( - id_tbl_dni, - tipo_documento, - num_documento, - id_solicitante - ) - VALUES ( - arreglo.usr_exist_datos_bas_id, - tipo_documento2, - upper(arreglo.usr_exist_datos_bas_cedula_pasaporte), - arreglo.usr_exist_datos_bas_id - ); - - raise notice 'Insertado el solicitante %',arreglo.usr_exist_datos_bas_id; - - else - - raise notice 'El Solicitante % % N°identidad % con el id: % ya se encuentra en la bd',arreglo.usr_exist_datos_bas_nombre, arreglo.usr_exist_datos_bas_apellido,arreglo.usr_exist_datos_bas_cedula_pasaporte,arreglo.usr_exist_datos_bas_id; - - end if; - - END LOOP; - - valor = 'Migración de datos basicos finalizada'; - - RETURN valor; - -END; -$BODY$ - LANGUAGE plpgsql VOLATILE - COST 100; diff --git a/postgresql/etl_examples/etl_descalificacion_medica().sql b/postgresql/etl_examples/etl_descalificacion_medica().sql deleted file mode 100755 index 2415e83..0000000 --- a/postgresql/etl_examples/etl_descalificacion_medica().sql +++ /dev/null @@ -1,78 +0,0 @@ -CREATE OR REPLACE FUNCTION etl_descalificacion_medica() - RETURNS character varying AS -$BODY$ - DECLARE - - - /* - Resumen: destinado a migrar los datos de la bd de origen a la nueva bd. - Se encarga de extraer los datos de la tabla lotus_usr_exist_descalif_med y los inserta en la tabla tbl_descalificacion_medica - ademas, actualiza el estatus_solicitante en la tabla tbl_documento asociados a todos los documentos correspondientes - al respectivo solicitante. - */ - - -- Extraccion de Datos de Descalificaciones Medicas-- - consulta5 cursor for select * from lotus_usr_exist_descalif_med; - arreglo lotus_usr_exist_descalif_med%ROWTYPE; - - arreglo_doc record; - - valor varchar; - - BEGIN - - FOR arreglo IN consulta5 LOOP - - -- Se verifica si ya existe el registro en la base de datos - if not exists(select id_descalificacion_lotus from tbl_descalificacion_medica where id_descalificacion_lotus = arreglo.usr_exist_descalif_med_id) then - - -- Se aplican las descalificaciones medicas a los correspondientes certificados (cap o cma) asociados - if (arreglo.tipo_descalif_med_id = 1) then - - update tbl_documento set descalificacion_medica = true, status_solicitante = 21, tx_motivo = arreglo.usr_exist_descalif_med_motivo where solicitante = arreglo.usr_exist_datos_bas_id; - - elsif (arreglo.tipo_descalif_med_id = 2) then - - update tbl_documento set descalificacion_medica = true, status_solicitante = 20, tx_motivo = arreglo.usr_exist_descalif_med_motivo where solicitante = arreglo.usr_exist_datos_bas_id; - - end if; - - FOR arreglo_doc IN select id_documento,solicitante from tbl_documento where solicitante = arreglo.usr_exist_datos_bas_id LOOP - - if (arreglo_doc.solicitante = arreglo.usr_exist_datos_bas_id) then - -- Transformacion y Carga de Datos -- - INSERT INTO tbl_descalificacion_medica ( - id_descalificacion_lotus, - documento, - tipo_descalif_medica, - motivo - ) - VALUES ( - arreglo.usr_exist_descalif_med_id, - arreglo_doc.id_documento, - arreglo.tipo_descalif_med_id, - arreglo.usr_exist_descalif_med_motivo - ); - - raise notice 'Inserte la descalificacion %, el documento fue % y el solicitante en desc fue %',arreglo.usr_exist_descalif_med_id,arreglo_doc.id_documento,arreglo.usr_exist_datos_bas_id; - - end if; - - end loop; - - else - - raise notice 'La descalificacion con id % ya se encuentra en la bd',arreglo.usr_exist_descalif_med_id; - - end if; - - valor = 'Migración de descalificaciones medicas finalizada'; - - END LOOP; - - RETURN valor; - -END; -$BODY$ - LANGUAGE plpgsql VOLATILE - COST 100; diff --git a/postgresql/etl_examples/etl_exp_cert.sql b/postgresql/etl_examples/etl_exp_cert.sql deleted file mode 100755 index 69e1e38..0000000 --- a/postgresql/etl_examples/etl_exp_cert.sql +++ /dev/null @@ -1,80 +0,0 @@ -CREATE OR REPLACE FUNCTION etl_exp_cert() - RETURNS character varying AS -$BODY$ - DECLARE - - /* - Resumen: destinado a migrar los datos de la bd de origen a la nueva bd. - Se encarga de extraer los datos de la tabla lotus_usr_exist_datos_exp_cert y los inserta en la tabla tbl_documento - */ - - -- Extraccion de los datos de los Certificados de Lotus - consulta2 cursor for select * from lotus_usr_exist_datos_exp_cert order by usr_exist_datos_exp_cert_fec_venc desc; - arreglo lotus_usr_exist_datos_exp_cert%ROWTYPE; - - d_num_documento varchar; - d_fecha_expedicion date; - d_fecha_vencimiento date; - valor varchar; - - BEGIN - - FOR arreglo IN consulta2 LOOP - - if not exists(select id_documento from tbl_documento where id_documento = arreglo.usr_exist_datos_bas_id) then - - select num_documento into strict d_num_documento from tbl_dni where id_solicitante = arreglo.usr_exist_datos_bas_id; - - -- Transformacion y Carga de Datos -- - - -- Se acomodan las fechas incorrectas - if(arreglo.usr_exist_datos_exp_cert_fec_exp = '%0001-01-01 BC%') then - d_fecha_expedicion = '0001-01-01'; - elsif(arreglo.usr_exist_datos_exp_cert_fec_venc = '%0001-01-01 BC%') then - d_fecha_vencimiento = '0001-01-01'; - elsif(arreglo.usr_exist_datos_exp_cert_fec_exp is null) then - d_fecha_expedicion = '0001-01-01'; - elsif(arreglo.usr_exist_datos_exp_cert_fec_venc is null) then - d_fecha_vencimiento = '0001-01-01'; - else - d_fecha_expedicion = arreglo.usr_exist_datos_exp_cert_fec_exp; - d_fecha_vencimiento = arreglo.usr_exist_datos_exp_cert_fec_venc; - end if; - - INSERT INTO tbl_documento ( - id_documento, - solicitante, - num_documento2, - clase, - fec_emic, - fec_vcto, - tipo_documento - ) - VALUES ( - arreglo.usr_exist_datos_exp_cert_id, - arreglo.usr_exist_datos_bas_id, - upper(d_num_documento), - arreglo.clase_cert_med_id, - d_fecha_expedicion, - d_fecha_vencimiento, - 2 - ); - - raise notice 'Insertando el Documento: %',arreglo.usr_exist_datos_exp_cert_id; - - else - - raise notice 'El Documento con id % ya se encuentra en la bd',arreglo.usr_exist_datos_exp_cert_id; - - end if; - - END LOOP; - - valor = 'Migración de expedientes de certificados finalizada'; - - RETURN valor; - -END; -$BODY$ - LANGUAGE plpgsql VOLATILE - COST 100; \ No newline at end of file diff --git a/postgresql/etl_examples/etl_habilitacion().sql b/postgresql/etl_examples/etl_habilitacion().sql deleted file mode 100755 index 87a3222..0000000 --- a/postgresql/etl_examples/etl_habilitacion().sql +++ /dev/null @@ -1,146 +0,0 @@ -CREATE OR REPLACE FUNCTION etl_habilitacion() - RETURNS character varying AS -$BODY$ - DECLARE - - - /* - Resumen: destinado a migrar los datos de la bd de origen a la nueva bd. - Se encarga de extraer los datos de la tabla lotus_usr_exist_datos_exp_hab y los inserta en la tabla tbl_habilitacion - */ - - d_funcion integer; --- d_clase integer; --- d_tipo integer; - d_verificar_tipo boolean; - d_verificar_clase boolean; - d_fecha_expedicion date; - d_fecha_vencimiento date; - - -- Extraccion de Datos -- - consulta4 cursor for select * from lotus_usr_exist_datos_exp_hab; - arreglo lotus_usr_exist_datos_exp_hab%ROWTYPE; - - i record; - - valor varchar; - - BEGIN - - FOR arreglo IN consulta4 LOOP - - if not exists(select id_habilitacion from tbl_habilitacion where id_habilitacion = arreglo.usr_exist_datos_exp_hab_id) then - for i in select usr_exist_datos_exp_lic_id from lotus_usr_exist_datos_exp_lic where usr_exist_datos_exp_lic_id = arreglo.usr_exist_datos_exp_lic_id loop - - -- Se verifica si la habilitacion es de clase - if (arreglo.tipo_habilitacion_id = 1) then - d_verificar_clase = true; - d_verificar_tipo = false; - - -- Se verifica si la habilitacion es de tipo - else - d_verificar_tipo = true; - d_verificar_clase = false; - - end if; - - -- Transformacion y Carga de Datos -- - - -- Se acomodan las fechas incorrectas - if(arreglo.usr_exist_datos_exp_hab_fec_exp = '%0001-01-01 BC%') then - d_fecha_expedicion = '0001-01-01'; - elsif(arreglo.usr_exist_datos_exp_hab_fec_venc = '%0001-01-01 BC%') then - d_fecha_vencimiento = '0001-01-01'; - elsif(arreglo.usr_exist_datos_exp_hab_fec_exp is null) then - d_fecha_expedicion = '0001-01-01'; - elsif(arreglo.usr_exist_datos_exp_hab_fec_venc is null) then - d_fecha_vencimiento = '0001-01-01'; - else - d_fecha_expedicion = arreglo.usr_exist_datos_exp_hab_fec_exp; - d_fecha_vencimiento = arreglo.usr_exist_datos_exp_hab_fec_venc; - end if; - - if (arreglo.tipo_habilitacion_id = 1) then - - INSERT INTO tbl_habilitacion ( - id_habilitacion, - clase, - tipo, - desc_habilitacion, - fec_habilitacion, - fec_vencimiento, - funcion, - tipo_licencia, - categoria, - verificar_clase, - verificar_tipo - ) - VALUES ( - arreglo.usr_exist_datos_exp_hab_id, - 1000000, - 1000000, - 'No Especificado', - d_fecha_expedicion, - d_fecha_vencimiento, - d_funcion, - arreglo.usr_exist_datos_exp_lic_id, - 100, - d_verificar_clase, - d_verificar_tipo - ); - - raise notice 'Inserte la habilitacion % y es de Clase',arreglo.usr_exist_datos_exp_hab_id; - - elsif (arreglo.tipo_habilitacion_id = 2) then - - -- Transformacion y Carga de Datos -- - INSERT INTO tbl_habilitacion ( - id_habilitacion, - clase, - tipo, - desc_habilitacion, - fec_habilitacion, - fec_vencimiento, - funcion, - tipo_licencia, - categoria, - verificar_clase, - verificar_tipo - ) - VALUES ( - arreglo.usr_exist_datos_exp_hab_id, - 1000000, - 1000000, - 'No Especificado', - d_fecha_expedicion, - d_fecha_vencimiento, - d_funcion, - arreglo.usr_exist_datos_exp_lic_id, - 100, - d_verificar_clase, - d_verificar_tipo - ); - - raise notice 'Inserte la habilitacion % y es de Tipo',arreglo.usr_exist_datos_exp_hab_id; - - end if; - - END LOOP; - - else - - raise notice 'La habilitacion con id % ya se encuentra en la bd',arreglo.usr_exist_datos_exp_hab_id; - - end if; - - - END LOOP; - - valor = 'Migración de habilitaciones finalizada'; - - RETURN valor; - -END; -$BODY$ - LANGUAGE plpgsql VOLATILE - COST 100; diff --git a/postgresql/etl_examples/etl_licencia.sql b/postgresql/etl_examples/etl_licencia.sql deleted file mode 100755 index 3f5bbc5..0000000 --- a/postgresql/etl_examples/etl_licencia.sql +++ /dev/null @@ -1,81 +0,0 @@ -CREATE OR REPLACE FUNCTION etl_licencia() - RETURNS character varying AS -$BODY$ - DECLARE - - /* - Resumen: destinado a migrar los datos de la bd de origen a la nueva bd. - Se encarga de extraer los datos de la tabla lotus_usr_exist_datos_exp_lic y los inserta en la tabla tbl_licencia - */ - - d_status2 int; - numero_licencia2 character varying(50); - d_fecha_expedicion date; - - -- Extraccion de Datos -- - consulta3 cursor for select * from lotus_usr_exist_datos_exp_lic; - arreglo lotus_usr_exist_datos_exp_lic%ROWTYPE; - - valor varchar; - - BEGIN - - FOR arreglo IN consulta3 LOOP - - if not exists(select id_licencia from tbl_licencia where id_licencia = arreglo.usr_exist_datos_exp_lic_id) then - - numero_licencia2 = arreglo.usr_exist_datos_exp_lic_numero; - - -- Transformacion y Carga de Datos -- - - -- Se homologan los 3 estatus de la bd lotus, haciendo correspondencia con los estatus cargados en la bd limed - if(arreglo.usr_exist_datos_exp_lic_estatus = '0') then -- No especificado -- - d_status2 = 0; - elsif(arreglo.usr_exist_datos_exp_lic_estatus = '1') then -- Vigente -- - d_status2 = 1; - elsif(arreglo.usr_exist_datos_exp_lic_estatus = '2') then -- Vencida -- - d_status2 = 2; - end if; - - -- Se acomodan las fechas incorrectas - if(arreglo.usr_exist_datos_exp_lic_fec_emi = '%0001-01-01 BC%') then - d_fecha_expedicion = '0001-01-01'; - elsif(arreglo.usr_exist_datos_exp_lic_fec_emi is null) then - d_fecha_expedicion = '0001-01-01'; - else - d_fecha_expedicion = arreglo.usr_exist_datos_exp_lic_fec_emi; - end if; - - INSERT INTO tbl_licencia ( - id_licencia, - id_solicitante, - fec_emision, - tipo_licencia, - status, - numero_licencia2 - ) - VALUES ( - arreglo.usr_exist_datos_exp_lic_id, - arreglo.usr_exist_datos_bas_id, - d_fecha_expedicion, - arreglo.tipo_pers_id, - d_status2, - upper(numero_licencia2) - ); - - else - - raise notice 'La licencia con id % ya se encuentra en la bd',arreglo.usr_exist_datos_exp_lic_id; - - end if; - - valor = 'Migración de licencias finalizada'; - - END LOOP; - - RETURN valor; - -END; -$BODY$ - LANGUAGE plpgsql VOLATILE - COST 100; diff --git a/postgresql/etl_examples/etl_migracion_general().sql b/postgresql/etl_examples/etl_migracion_general().sql deleted file mode 100755 index cb0749a..0000000 --- a/postgresql/etl_examples/etl_migracion_general().sql +++ /dev/null @@ -1,44 +0,0 @@ -CREATE OR REPLACE FUNCTION etl_migracion_general() - RETURNS character varying AS -$BODY$ - DECLARE - - /* Migración de Datos usada para trasladar los datos de un proyecto del sector aeronáutico - - Para probarlo: select etl_migracion_general() - - - Para eliminar datos de lotus: - - delete from lotus_usr_exist_datos_basicos; - delete from lotus_usr_exist_datos_exp_cert; - delete from lotus_usr_exist_datos_exp_hab; - delete from lotus_usr_exist_datos_exp_lic; - delete from lotus_usr_exist_descalif_med; - - */ - - -- Para valores de retorno - resultado_etl_datos_basicos varchar; - resultado_etl_exp_cert varchar; - resultado_etl_descalificaciones varchar; - resultado_etl_licencia varchar; - resultado_etl_habilitaciones varchar; - - valor varchar; - - BEGIN - - resultado_etl_datos_basicos = (select etl_datos_basicos()); - resultado_etl_exp_cert = (select etl_exp_cert()); - resultado_etl_descalificaciones = (select etl_descalificacion_medica()); - resultado_etl_licencia = (select etl_licencia()); - resultado_etl_habilitaciones = (select etl_habilitacion()); - - valor = 'Migración de datos finalizada'; - - RETURN valor; - -END; -$BODY$ - LANGUAGE plpgsql VOLATILE - COST 100; From 07097610834e82e81e45ee0bca772b16d70d51f6 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Tue, 9 Jul 2019 22:28:27 -0400 Subject: [PATCH 12/24] Adding call to some sql functions in execute_all.sh file --- execute_all.sh | 17 +-- .../temporal_schema/data/tbl_categories.sql | 2 - .../data/tbl_endangered_animals.sql | 4 - .../temporal_schema/data/tbl_reasons.sql | 4 - .../data/tbl_relation_endangered_reasons.sql | 6 - .../functions/udf_endangered_animals_get.sql | 65 ---------- ...df_endangered_animals_reasons_tree_get.sql | 120 ------------------ .../temporal_schema/tables/tbl_categories.sql | 12 -- .../tables/tbl_endangered_animals.sql | 15 --- .../tables/tbl_log_categories.sql | 11 -- .../tables/tbl_log_endangered_animals.sql | 14 -- .../tables/tbl_log_reasons.sql | 11 -- .../tbl_log_relation_endangered_reasons.sql | 11 -- .../temporal_schema/tables/tbl_reasons.sql | 12 -- .../tbl_relation_endangered_reasons.sql | 22 ---- 15 files changed, 2 insertions(+), 324 deletions(-) delete mode 100644 postgresql/dev_postgres_database/temporal_schema/data/tbl_categories.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/data/tbl_endangered_animals.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/data/tbl_reasons.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/data/tbl_relation_endangered_reasons.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/functions/udf_endangered_animals_get.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/functions/udf_endangered_animals_reasons_tree_get.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/tables/tbl_categories.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/tables/tbl_endangered_animals.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_categories.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_endangered_animals.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_reasons.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_relation_endangered_reasons.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/tables/tbl_reasons.sql delete mode 100644 postgresql/dev_postgres_database/temporal_schema/tables/tbl_relation_endangered_reasons.sql diff --git a/execute_all.sh b/execute_all.sh index 74fc181..18dda9f 100755 --- a/execute_all.sh +++ b/execute_all.sh @@ -1,19 +1,6 @@ psql -U postgres -a -f ./postgresql/main_database.sql psql -U postgres -d dev_postgres_database -a -c "CREATE SCHEMA temporal_schema;" -#psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udt_general_tables_insert.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/tables/tbl_categories.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_categories.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/tables/tbl_endangered_animals.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_endangered_animals.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/tables/tbl_reasons.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_reasons.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/tables/tbl_relation_endangered_reasons.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_relation_endangered_reasons.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/data/tbl_categories.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/data/tbl_endangered_animals.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/data/tbl_reasons.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/data/tbl_relation_endangered_reasons.sql psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udf_kill_connections_pg3.sql psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udf_kill_connections_pg4.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udf_endangered_animals_get.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udf_endangered_animals_reasons_tree_get.sql +psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udf_check_value.sql +psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udf_adjust_sequence.sql \ No newline at end of file diff --git a/postgresql/dev_postgres_database/temporal_schema/data/tbl_categories.sql b/postgresql/dev_postgres_database/temporal_schema/data/tbl_categories.sql deleted file mode 100644 index 039eb4d..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/data/tbl_categories.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT INTO temporal_schema.tbl_categories (name,last_modified_date) VALUES ('Mammals',NOW()); -INSERT INTO temporal_schema.tbl_categories (name,last_modified_date) VALUES ('Reptile',NOW()); \ No newline at end of file diff --git a/postgresql/dev_postgres_database/temporal_schema/data/tbl_endangered_animals.sql b/postgresql/dev_postgres_database/temporal_schema/data/tbl_endangered_animals.sql deleted file mode 100644 index 98f1fc5..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/data/tbl_endangered_animals.sql +++ /dev/null @@ -1,4 +0,0 @@ -INSERT INTO temporal_schema.tbl_endangered_animals (name,id_categories,last_modified_date) VALUES ('Bengal tiger',1,NOW()); -INSERT INTO temporal_schema.tbl_endangered_animals (name,id_categories,last_modified_date) VALUES ('Giant panda',1,NOW()); -INSERT INTO temporal_schema.tbl_endangered_animals (name,id_categories,last_modified_date) VALUES ('Green Turtle',2,NOW()); -INSERT INTO temporal_schema.tbl_endangered_animals (name,id_categories,last_modified_date) VALUES ('Hippopotamus',1,NOW()); diff --git a/postgresql/dev_postgres_database/temporal_schema/data/tbl_reasons.sql b/postgresql/dev_postgres_database/temporal_schema/data/tbl_reasons.sql deleted file mode 100644 index f1042b3..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/data/tbl_reasons.sql +++ /dev/null @@ -1,4 +0,0 @@ -INSERT INTO temporal_schema.tbl_reasons (description,last_modified_date) VALUES ('Poaching',NOW()); -INSERT INTO temporal_schema.tbl_reasons (description,last_modified_date) VALUES ('Sales',NOW()); -INSERT INTO temporal_schema.tbl_reasons (description,last_modified_date) VALUES ('Sea contamination',NOW()); -INSERT INTO temporal_schema.tbl_reasons (description,last_modified_date) VALUES ('Destruction of territory',NOW()); diff --git a/postgresql/dev_postgres_database/temporal_schema/data/tbl_relation_endangered_reasons.sql b/postgresql/dev_postgres_database/temporal_schema/data/tbl_relation_endangered_reasons.sql deleted file mode 100644 index 357e00a..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/data/tbl_relation_endangered_reasons.sql +++ /dev/null @@ -1,6 +0,0 @@ -INSERT INTO temporal_schema.tbl_relation_endangered_reasons (id_endangered_animals,id_categories,id_reasons,last_modified_date) VALUES (1,1,1,NOW()); -INSERT INTO temporal_schema.tbl_relation_endangered_reasons (id_endangered_animals,id_categories,id_reasons,last_modified_date) VALUES (1,1,2,NOW()); -INSERT INTO temporal_schema.tbl_relation_endangered_reasons (id_endangered_animals,id_categories,id_reasons,last_modified_date) VALUES (2,1,1,NOW()); -INSERT INTO temporal_schema.tbl_relation_endangered_reasons (id_endangered_animals,id_categories,id_reasons,last_modified_date) VALUES (3,2,3,NOW()); -INSERT INTO temporal_schema.tbl_relation_endangered_reasons (id_endangered_animals,id_categories,id_reasons,last_modified_date) VALUES (4,1,1,NOW()); -INSERT INTO temporal_schema.tbl_relation_endangered_reasons (id_endangered_animals,id_categories,id_reasons,last_modified_date) VALUES (4,1,4,NOW()); diff --git a/postgresql/dev_postgres_database/temporal_schema/functions/udf_endangered_animals_get.sql b/postgresql/dev_postgres_database/temporal_schema/functions/udf_endangered_animals_get.sql deleted file mode 100644 index 57d778e..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/functions/udf_endangered_animals_get.sql +++ /dev/null @@ -1,65 +0,0 @@ --- DROP FUNCTION temporal_schema.udf_endangered_animals_get(INTEGER); - -CREATE OR REPLACE FUNCTION temporal_schema.udf_endangered_animals_get(param_id_endangered_animals INTEGER) - RETURNS character varying - LANGUAGE plpgsql -AS $function$ - -DECLARE - -- This is a comment: in this section you can declare your variables - - local_returning VARCHAR(200) = ''; - - BEGIN - - /* It is a suggest of a good header - Author: - Creation Date: - Review: - - -- To Test: you can use the next line, to test this function - SELECT temporal_schema.udf_endangered_animals_get(1); - */ - - -- This is a validation to check if the param_id_endangered_animals exist - IF EXISTS ( - SELECT - name - FROM - temporal_schema.tbl_endangered_animals - WHERE - id_endangered_animals = param_id_endangered_animals - AND - is_active - AND - not is_deleted - ) THEN - - -- You can use 'INTO STRICT' statement, to store data returned into a variable. Be carefull if your query return more than one values - SELECT - name INTO STRICT local_returning - FROM - temporal_schema.tbl_endangered_animals - WHERE - id_endangered_animals = param_id_endangered_animals - AND - is_active - AND - not is_deleted; - - -- You can use 'RAISE NOTICE' statement to display messages and debug your code - RAISE NOTICE 'Data found'; - - ELSE - - RAISE NOTICE 'Not data found'; - - END IF; - - RETURN local_returning; - -END; - -$function$; - -COMMENT ON FUNCTION temporal_schema.udf_endangered_animals_get(INTEGER) IS 'This function is used to get data from temporal_schema.udf_endangered_animals_get'; diff --git a/postgresql/dev_postgres_database/temporal_schema/functions/udf_endangered_animals_reasons_tree_get.sql b/postgresql/dev_postgres_database/temporal_schema/functions/udf_endangered_animals_reasons_tree_get.sql deleted file mode 100644 index 1fa9206..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/functions/udf_endangered_animals_reasons_tree_get.sql +++ /dev/null @@ -1,120 +0,0 @@ --- DROP FUNCTION temporal_schema.udf_endangered_animals_reasons_tree_get(INTEGER); - -CREATE OR REPLACE FUNCTION temporal_schema.udf_endangered_animals_reasons_tree_get() - RETURNS json - LANGUAGE plpgsql -AS $function$ - -DECLARE - - json_returning json = '{}'; - - BEGIN - - /* - -- To Test: - SELECT temporal_schema.udf_endangered_animals_reasons_tree_get(); - */ - - WITH temporal_category_table AS ( - SELECT - id_categories, - name, - is_active, - is_deleted - FROM - temporal_schema.tbl_categories - WHERE - is_active - AND - not is_deleted - ), temporal_relation_endangered_reasons AS ( - SELECT - id_endangered_animals, - id_categories, - id_reasons, - is_active, - is_deleted - FROM - temporal_schema.tbl_relation_endangered_reasons - WHERE - is_active - AND - not is_deleted - ) - - SELECT JSON_AGG(a.*) INTO STRICT json_returning - FROM ( - SELECT - tct.id_categories "categoryId", - tct.name "categoryName", - true as "isActive", - ( - SELECT ARRAY_AGG(b.*) "animalList" - FROM ( - SELECT - -- DISTINCT ON (ea.id_endangered_animals) -- it also could be, distinct on rather than group by - ea.id_endangered_animals "animalId", - ea.name "animalName", - true as "isActive", - ( - SELECT JSON_AGG(c.*) "endangeredReasonsList" - FROM ( - SELECT - r.id_reasons "reasonId", - r.description "reason", - true as "isActive" - FROM temporal_schema.tbl_reasons r - INNER JOIN temporal_relation_endangered_reasons trer - ON r.id_reasons = trer.id_reasons - AND - trer.id_endangered_animals = ea.id_endangered_animals - AND - trer.id_categories = tct.id_categories - AND - r.is_active - AND - not r.is_deleted - AND - trer.is_active - AND - not trer.is_deleted - )c - ) - FROM temporal_schema.tbl_endangered_animals ea - INNER JOIN temporal_schema.tbl_relation_endangered_reasons rer - ON ea.id_endangered_animals = rer.id_endangered_animals - AND - ea.id_categories = rer.id_categories - AND - ea.id_categories = tct.id_categories - AND - ea.is_active - AND - not ea.is_deleted - AND - rer.is_active - AND - not rer.is_deleted - GROUP BY - ea.id_endangered_animals, - ea.name - ORDER BY - ea.id_endangered_animals - )b - ) - FROM - temporal_category_table tct - WHERE - tct.is_active - AND - not tct.is_deleted - )a; - - RETURN json_returning; - -END; - -$function$; - -COMMENT ON FUNCTION temporal_schema.udf_endangered_animals_reasons_tree_get() IS 'This function is used to get a tree of endangered animals with endangered reasons.'; diff --git a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_categories.sql b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_categories.sql deleted file mode 100644 index a5336a8..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_categories.sql +++ /dev/null @@ -1,12 +0,0 @@ ---DROP TABLE IF EXISTS temporal_schema.tbl_categories; -CREATE TABLE temporal_schema.tbl_categories ( - id_categories SERIAL NOT NULL PRIMARY KEY, - name VARCHAR(255), - is_active BOOLEAN DEFAULT true, - is_deleted BOOLEAN DEFAULT false, - last_modified_date timestamp with time zone NOT NULL -); -/*CREATE TRIGGER trigger_categories_insert BEFORE UPDATE OR DELETE ON temporal_schema.tbl_categories - FOR EACH ROW - EXECUTE PROCEDURE udt_general_tables_insert(); -*/ \ No newline at end of file diff --git a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_endangered_animals.sql b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_endangered_animals.sql deleted file mode 100644 index e17df9e..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_endangered_animals.sql +++ /dev/null @@ -1,15 +0,0 @@ ---DROP TABLE IF EXISTS temporal_schema.tbl_endangered_animals; -CREATE TABLE temporal_schema.tbl_endangered_animals ( - id_endangered_animals SERIAL NOT NULL, - name VARCHAR(255), - id_categories INTEGER, - is_active BOOLEAN DEFAULT true, - is_deleted BOOLEAN DEFAULT false, - last_modified_date timestamp with time zone NOT NULL, - created_date timestamp with time zone DEFAULT NOW(), - PRIMARY KEY (id_endangered_animals,id_categories) -); -/*CREATE TRIGGER trigger_endangered_animals_insert BEFORE UPDATE OR DELETE ON temporal_schema.tbl_endangered_animals - FOR EACH ROW - EXECUTE PROCEDURE udt_general_tables_insert(); -*/ \ No newline at end of file diff --git a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_categories.sql b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_categories.sql deleted file mode 100644 index 5608645..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_categories.sql +++ /dev/null @@ -1,11 +0,0 @@ ---DROP TABLE IF EXISTS temporal_schema.tbl_log_categories; -CREATE TABLE temporal_schema.tbl_log_categories ( - id_categories INTEGER, - name VARCHAR(255), - is_active BOOLEAN DEFAULT true, - is_deleted BOOLEAN DEFAULT false, - last_modified_date timestamp with time zone NOT NULL, - last_modified_by CHARACTER VARYING(100), - ip_user CHARACTER VARYING(100), - update_or_deleted_date timestamp with time zone NOT NULL -); \ No newline at end of file diff --git a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_endangered_animals.sql b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_endangered_animals.sql deleted file mode 100644 index d45c940..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_endangered_animals.sql +++ /dev/null @@ -1,14 +0,0 @@ ---DROP TABLE IF EXISTS temporal_schema.tbl_log_endangered_animals; -CREATE TABLE temporal_schema.tbl_log_endangered_animals ( - id_endangered_animals INTEGER, - name VARCHAR(255), - is_active BOOLEAN DEFAULT true, - is_deleted BOOLEAN DEFAULT false, - last_modified_date timestamp with time zone NOT NULL, - last_modified_by CHARACTER VARYING(100), - ip_user CHARACTER VARYING(100), - update_or_deleted_date timestamp with time zone NOT NULL -); -ALTER TABLE temporal_schema.tbl_endangered_animals ADD CONSTRAINT tbll_endangered_animals_tb_categories - FOREIGN KEY (id_categories) REFERENCES temporal_schema.tbl_categories (id_categories) -ON UPDATE CASCADE; \ No newline at end of file diff --git a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_reasons.sql b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_reasons.sql deleted file mode 100644 index 536073c..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_reasons.sql +++ /dev/null @@ -1,11 +0,0 @@ ---DROP TABLE IF EXISTS temporal_schema.tbl_log_reasons; -CREATE TABLE temporal_schema.tbl_log_reasons ( - id_reasons INTEGER, - name VARCHAR(255), - is_active BOOLEAN DEFAULT true, - is_deleted BOOLEAN DEFAULT false, - last_modified_date timestamp with time zone NOT NULL, - last_modified_by CHARACTER VARYING(100), - ip_user CHARACTER VARYING(100), - update_or_deleted_date timestamp with time zone NOT NULL -); \ No newline at end of file diff --git a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_relation_endangered_reasons.sql b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_relation_endangered_reasons.sql deleted file mode 100644 index af5ff22..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_log_relation_endangered_reasons.sql +++ /dev/null @@ -1,11 +0,0 @@ ---DROP TABLE IF EXISTS temporal_schema.tbl_log_relation_endangered_reasons; -CREATE TABLE temporal_schema.tbl_log_relation_endangered_reasons ( - id_relation_endangered_reasons INTEGER, - name VARCHAR(255), - is_active BOOLEAN DEFAULT true, - is_deleted BOOLEAN DEFAULT false, - last_modified_date timestamp with time zone NOT NULL, - last_modified_by CHARACTER VARYING(100), - ip_user CHARACTER VARYING(100), - update_or_deleted_date timestamp with time zone NOT NULL -); \ No newline at end of file diff --git a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_reasons.sql b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_reasons.sql deleted file mode 100644 index b256e3f..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_reasons.sql +++ /dev/null @@ -1,12 +0,0 @@ ---DROP TABLE IF EXISTS temporal_schema.tbl_reasons; -CREATE TABLE temporal_schema.tbl_reasons ( - id_reasons SERIAL NOT NULL PRIMARY KEY, - description VARCHAR(255), - is_active BOOLEAN DEFAULT true, - is_deleted BOOLEAN DEFAULT false, - last_modified_date timestamp with time zone NOT NULL -); -/*CREATE TRIGGER trigger_reasons_insert BEFORE UPDATE OR DELETE ON temporal_schema.tbl_reasons - FOR EACH ROW - EXECUTE PROCEDURE udt_general_tables_insert(); -*/ \ No newline at end of file diff --git a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_relation_endangered_reasons.sql b/postgresql/dev_postgres_database/temporal_schema/tables/tbl_relation_endangered_reasons.sql deleted file mode 100644 index 71f8555..0000000 --- a/postgresql/dev_postgres_database/temporal_schema/tables/tbl_relation_endangered_reasons.sql +++ /dev/null @@ -1,22 +0,0 @@ ---DROP TABLE IF EXISTS temporal_schema.tbl_relation_endangered_reasons; -CREATE TABLE temporal_schema.tbl_relation_endangered_reasons ( - id_endangered_reasons_relation SERIAL, - id_endangered_animals INTEGER NOT NULL, - id_categories INTEGER NOT NULL, - id_reasons INTEGER NOT NULL, - is_active BOOLEAN DEFAULT true, - is_deleted BOOLEAN DEFAULT false, - last_modified_date timestamp with time zone NOT NULL, - created_date timestamp with time zone DEFAULT NOW(), - PRIMARY KEY (id_endangered_reasons_relation,id_categories,id_endangered_animals,id_reasons) -); -ALTER TABLE temporal_schema.tbl_relation_endangered_reasons ADD CONSTRAINT tbl_relation_endangered_reasons_tbl_reasons - FOREIGN KEY (id_reasons) REFERENCES temporal_schema.tbl_reasons (id_reasons) -ON UPDATE CASCADE; -ALTER TABLE temporal_schema.tbl_relation_endangered_reasons ADD CONSTRAINT tbl_relation_endangered_reasons_tbl_endangered_animals - FOREIGN KEY (id_endangered_animals,id_categories) REFERENCES temporal_schema.tbl_endangered_animals (id_endangered_animals,id_categories) -ON UPDATE CASCADE; -/*CREATE TRIGGER trigger_relation_endangered_reasons_insert BEFORE UPDATE OR DELETE ON temporal_schema.tbl_relation_endangered_reasons - FOR EACH ROW - EXECUTE PROCEDURE udt_general_tables_insert(); -*/ \ No newline at end of file From 4198e8d62d0f07de7e90955330557e1fb6904146 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Tue, 9 Jul 2019 22:57:43 -0400 Subject: [PATCH 13/24] Execute all modified --- execute_all.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/execute_all.sh b/execute_all.sh index 18dda9f..b65a65c 100755 --- a/execute_all.sh +++ b/execute_all.sh @@ -1,6 +1,5 @@ psql -U postgres -a -f ./postgresql/main_database.sql -psql -U postgres -d dev_postgres_database -a -c "CREATE SCHEMA temporal_schema;" -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udf_kill_connections_pg3.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udf_kill_connections_pg4.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udf_check_value.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/temporal_schema/functions/udf_adjust_sequence.sql \ No newline at end of file +psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/public/functions/udf_kill_connections_pg3.sql +psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/public/functions/udf_kill_connections_pg4.sql +psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/public/functions/udf_check_value.sql +psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/public/functions/udf_adjust_sequence.sql \ No newline at end of file From 99e0a64c7b21779740e7d1457f35543b21dec29b Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Tue, 9 Jul 2019 23:05:18 -0400 Subject: [PATCH 14/24] Adding lists of queries and scripts --- README.md | 54 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index d6cdbcf..41df0bb 100644 --- a/README.md +++ b/README.md @@ -18,25 +18,41 @@ And execute the execute_all.sh file: ## Structure: -* dev_postgres_database folder: - -Files are contained inside of the dev_postgres_database folder, with the below structure: - -a) schema -> useful_queries: - -- dba_scripts.sql: some dba monitoring querys. - -- useful_structure_queries.sql: several useful querys related with schemas, sequences, tables, columns, triggers, constraints, functions and pg_notify channels. - -b) schema -> functions: several PL/PL/PgSQL functions with useful code (e.g: generating queries with tree json format, killing connections in pgadmin and more). - -c) schema -> tables: examples tables to use in the plpgsql_scripts. - -d) schema -> data: mock data to use in the plpgsql_scripts. - -* etl_examples folder: - -Some etl examples developed in PL/PgSQL, with useful code (e.g cursors, data cleaning and more) +Files are contained inside of the dev_postgres_database folder, with the follow structure: dev_postgres_database -> schema -> functions. + +1. dba_scripts.sql: + +- Display actives connections +- Display the activity of your database +- Display connections by client_addr +- Display usaged size on disk by database +- Display usaged size of a specific table +- Display usaged size of tables by schemas + +2. udf_check_value.sql, udf_adjust_sequence.sql: scripts to adjust sequences. + +3. udf_kill_connections_pg3.sql, udf_kill_connections_pg4.sql: scripts used to controll the excessive number of queries windows opened in pgadmin 3 and 4. + +4. useful_structure_queries.sql: several useful querys related with schemas, sequences, tables, columns, triggers, constraints, functions and pg_notify channels. + +- List schemas names +- List all sequences information +- Display all the information about an specific sequence +- List sequences comments +- List all the tables names of an schema +- List all the tables of your database with their primary keys +- List all the tables indexs +- List all columns for a specified table +- List all columns for a specified table (another option) +- Search an specific column excluding postgres pg_catalogs +- List all the triggers +- List all the triggers of a schema +- List all the foreign keys +- List all the functions including postgres +- List all the functions of a schema +- Search (functions, tables, colums...) in all the functions (Great function!) +- List function comments of a schema +- List current channels listening Contributions ----------------------- From fed1c71277a87f6964e89ffb76f1db12e9cd76e1 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Tue, 23 Jul 2019 22:43:31 -0400 Subject: [PATCH 15/24] Adding useful PostGIS material and queries --- .../functions/useful_postgis_queries.sql | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql diff --git a/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql b/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql new file mode 100644 index 0000000..84dedfd --- /dev/null +++ b/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql @@ -0,0 +1,122 @@ +-- PostGIS tests + +----------------------------------------------- +-- ABOUT INSTALLATION +----------------------------------------------- + +-- Default installation +CREATE EXTENSION postgis; + +/* +-- See more about the installation and aditional extensions here: + +http://postgis.net/install/ + +https://postgis.net/docs/postgis_installation.html#idm559 + +*/ + +----------------------------------------------- +-- THINGS YOU SHOULD KNOW RELATED WITH GEOLOCALIZATION AND BEFORE TO WORK WITH POSTGIS +----------------------------------------------- + +/* +-- WGS 84 (World Geodetic System 1984) +https://es.wikipedia.org/wiki/WGS84 + +-- What are EPSG / SRID codes and their link with PostGIS +https://mappinggis.com/2016/04/los-codigos-epsg-srid-vinculacion-postgis/ + +-- Spatial reference system +https://en.wikipedia.org/wiki/Spatial_reference_system + +-- Introduction to Spatial References (See 4326 - GPS section) +https://developers.arcgis.com/documentation/core-concepts/spatial-references/ + +-- Reverse Geocoding +https://en.wikipedia.org/wiki/Reverse_geocoding + +-- Admin level reference (useful for reverse geocoding queries, not only for PostGIS) +https://wiki.openstreetmap.org/wiki/Nominatim/Development_overview#Country_to_street_level + +-- Nearest-Neighbour Searching +https://postgis.net/workshops/postgis-intro/knn.html + +*/ + +----------------------------------------------- +-- ABOUT DOCUMENTATION +----------------------------------------------- + +/* +-- Official Documentation +http://postgis.net/docs/ + +-- PostGIS 2.5.3dev Manual +http://postgis.net/stuff/postgis-2.5.pdf + +-- PostGIS reference for functions description +http://postgis.net/docs/reference.html + +-- FAQ: +https://postgis.net/docs/PostGIS_FAQ.html + +http://postgis.net/2013/08/30/tip_ST_Set_or_Transform/ + +*/ + +----------------------------------------------- +-- USEFUL QUERIES +----------------------------------------------- + +-- Adding a geometry columns: +ALTER TABLE + ADD COLUMN geom geometry(Geometry,4326); + +ALTER TABLE + ADD COLUMN position geometry(Point,4326); + +-- Traduce your lat and long columns in a geometry point based on WGS 84 +UPDATE + SET = ST_SetSRID(ST_MakePoint(,),4326); + +-- Distance between 2 points (longitude latitude) based on WGS 84 +SELECT ST_Distance( + ST_GeometryFromText('POINT(-118.4079 33.9434)', 4326), -- Los Angeles (LAX) + ST_GeometryFromText('POINT(2.5559 49.0083)', 4326) -- Paris (CDG) + ); + +-- Get X,Y coordinates from latitude and longitude +SELECT + ST_X() latitude, + ST_Y() longitude, + ST_AsText() +FROM + ; + +-- Get geometry value from text (longitude latitude) +SELECT + ST_GeogFromText('SRID=4267;POINT(-77.0092 38.889588)'); + +-- Get geometry value from text (longitude latitude) +SELECT + ST_Transform(.,4269) +FROM + ; + +-- Find intersection based on WGS 84 +SELECT + +FROM + +WHERE + ST_intersects(ST_GeographyFromText('SRID=4326;POINT(' || . || ' ' || . || ')'), .) + +# Find near by places within 5 km from a current position(longitude,latitude) +SELECT + * +FROM + +WHERE + ST_DistanceSphere(""."", ST_GeomFromEWKB(ST_MakePoint(,)::bytea)) <= 5000.0 + From 67dfbdcee92cae988273b4bd1cd3283116af9ec0 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Tue, 23 Jul 2019 23:38:40 -0400 Subject: [PATCH 16/24] Adding list of queries and official repositories --- README.md | 8 ++++++++ .../public/functions/useful_postgis_queries.sql | 13 +++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 41df0bb..1c537da 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,14 @@ Files are contained inside of the dev_postgres_database folder, with the follow - List function comments of a schema - List current channels listening +5. useful_postgis_queries.sql: several useful querys related with geolocation using postgis extension. + +- Distance between 2 points (longitude latitude) based on WGS 84 +- Get X,Y coordinates from geometry point +- Get geometry value from text (longitude latitude) +- Find intersection based on WGS 84 +- Find near by places within X km from a current position(longitude,latitude) + Contributions ----------------------- diff --git a/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql b/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql index 84dedfd..5d1a5ac 100644 --- a/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql +++ b/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql @@ -49,6 +49,7 @@ https://postgis.net/workshops/postgis-intro/knn.html ----------------------------------------------- /* + -- Official Documentation http://postgis.net/docs/ @@ -63,6 +64,14 @@ https://postgis.net/docs/PostGIS_FAQ.html http://postgis.net/2013/08/30/tip_ST_Set_or_Transform/ +-- PostGIS official repository: +https://trac.osgeo.org/postgis/ + +https://github.com/postgis/postgis + +-- pgRouting official repository +https://github.com/pgRouting/pgrouting + */ ----------------------------------------------- @@ -86,7 +95,7 @@ SELECT ST_Distance( ST_GeometryFromText('POINT(2.5559 49.0083)', 4326) -- Paris (CDG) ); --- Get X,Y coordinates from latitude and longitude +-- Get X,Y coordinates from geometry point SELECT ST_X() latitude, ST_Y() longitude, @@ -112,7 +121,7 @@ FROM WHERE ST_intersects(ST_GeographyFromText('SRID=4326;POINT(' || . || ' ' || . || ')'), .) -# Find near by places within 5 km from a current position(longitude,latitude) +-- Find near by places within 5 km from a current position(longitude,latitude) SELECT * FROM From c6e242b42eeadbd283ebc0ffe5111101055ad1f9 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Fri, 26 Jul 2019 22:58:55 -0400 Subject: [PATCH 17/24] Adding query relate with <-> operator in PostGIS --- .../public/functions/useful_postgis_queries.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql b/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql index 5d1a5ac..dfdca05 100644 --- a/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql +++ b/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql @@ -129,3 +129,10 @@ FROM WHERE ST_DistanceSphere(""."", ST_GeomFromEWKB(ST_MakePoint(,)::bytea)) <= 5000.0 +-- Get the first 5 nearest records disregard (<->) how far the are away from the current location (latitude and longitude as a parameters) +SELECT + * +FROM + +ORDER BY + . <-> ST_SetSRID(ST_MakePoint(param_lng,param_lat),4326) LIMIT 5; From 1b6b1f405ee7727142c52db53be6cb8e97598c30 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Tue, 3 Dec 2019 23:59:58 -0400 Subject: [PATCH 18/24] Adding basic grid clustering query for multiples nearby places --- .../public/functions/useful_postgis_queries.sql | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql b/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql index dfdca05..69aa8cb 100644 --- a/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql +++ b/postgresql/dev_postgres_database/public/functions/useful_postgis_queries.sql @@ -136,3 +136,16 @@ FROM ORDER BY . <-> ST_SetSRID(ST_MakePoint(param_lng,param_lat),4326) LIMIT 5; + + +-- Basic grid clustering with SnapToGrid +SELECT + array_agg(id) AS ids, + COUNT( position ) AS count, + ST_AsText( ST_Centroid(ST_Collect( position )) ) AS center +FROM + +GROUP BY + ST_SnapToGrid( ST_SetSRID(position, 4326), 22.25, 11.125) +ORDER BY + count DESC; \ No newline at end of file From e924ac6ddc45936ab34e6cdd7ca2b3d25b8f9b64 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 21 Mar 2020 13:15:23 -0400 Subject: [PATCH 19/24] Adding example to do full text search (fts) --- .../public/functions/useful_fts_queries.sql | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql diff --git a/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql b/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql new file mode 100644 index 0000000..3bb00bf --- /dev/null +++ b/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql @@ -0,0 +1,31 @@ +-- To search incoming string in a column + +/* +1. Remove white spaces at the begining or at the end with ```trim()``` +2. Append ```:*``` at the end of the string to allow to search incomplete string +3. Replace space between multiple words with ```&``` to allow to search incomplete string with multiple words +4. Apply ```lower()``` function +5. Remove accents with ```UNACCENT()```, previously installed with: +``` CREATE EXTENSION unaccent; +``` +6. Remove (spanish) stop words with ```to_tsquery()``` +7. Match string with the column selected + +* Important Note: consider that the column should have some index (GIN, GiST or B-tree, see more information here: https://www.postgresql.org/docs/current/textsearch-indexes.html) to make improvements in fast searchs + +* To check efficient of this query, use: +``` +EXPLAIN ANALYZE query +``` +See important documentation below (according with your postgresql version): +- https://www.postgresql.org/docs/current/textsearch.html +- https://www.postgresql.org/docs/11/pgtrgm.html +- https://stackoverflow.com/questions/2513501/postgresql-full-text-search-how-to-search-partial-words +*/ + +SELECT + * +FROM + +WHERE + (to_tsvector('spanish',UNACCENT(lower(place_name))) @@ to_tsquery('spanish',UNACCENT(lower(replace(trim(param_string) || ':*',' ','&'))))) or From 62cfcde13612a38a2d4144fc4685af94f90e7731 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 21 Mar 2020 15:17:29 -0400 Subject: [PATCH 20/24] Adding table,data,examples of searchs and more information related with fts --- execute_all.sh | 4 +- .../public/data/places.sql | 3 ++ .../public/functions/useful_fts_queries.sql | 49 +++++++++++++++++-- .../public/tables/places.sql | 8 +++ 4 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 postgresql/dev_postgres_database/public/data/places.sql create mode 100644 postgresql/dev_postgres_database/public/tables/places.sql diff --git a/execute_all.sh b/execute_all.sh index b65a65c..36a2e04 100755 --- a/execute_all.sh +++ b/execute_all.sh @@ -2,4 +2,6 @@ psql -U postgres -a -f ./postgresql/main_database.sql psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/public/functions/udf_kill_connections_pg3.sql psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/public/functions/udf_kill_connections_pg4.sql psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/public/functions/udf_check_value.sql -psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/public/functions/udf_adjust_sequence.sql \ No newline at end of file +psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/public/functions/udf_adjust_sequence.sql +psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/public/tables/places.sql +psql -U postgres -d dev_postgres_database -a -f ./postgresql/dev_postgres_database/public/data/places.sql \ No newline at end of file diff --git a/postgresql/dev_postgres_database/public/data/places.sql b/postgresql/dev_postgres_database/public/data/places.sql new file mode 100644 index 0000000..25736ed --- /dev/null +++ b/postgresql/dev_postgres_database/public/data/places.sql @@ -0,0 +1,3 @@ +INSERT INTO public.places (id, name, created_at, updated_at, is_active, is_deleted) VALUES (1, 'El Salto Ángel', now(), now(), true, false); +INSERT INTO public.places (id, name, created_at, updated_at, is_active, is_deleted) VALUES (2, 'The Wizarding World of Harry Potter', now(), now(), true, false); +INSERT INTO public.places (id, name, created_at, updated_at, is_active, is_deleted) VALUES (3, 'Triángulo de las Bermudas', now(), now(), true, false); diff --git a/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql b/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql index 3bb00bf..5e71973 100644 --- a/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql +++ b/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql @@ -1,6 +1,16 @@ -- To search incoming string in a column +SELECT + * +FROM + +WHERE + (to_tsvector('spanish',UNACCENT(lower())) @@ to_tsquery('spanish',UNACCENT(lower(replace(trim() || ':*',' ','&'))))) or + (to_tsvector('spanish',UNACCENT(lower())) @@ to_tsquery('spanish',UNACCENT(lower(replace(trim() || ':*',' ','&'))))) + + /* +# Steps: 1. Remove white spaces at the begining or at the end with ```trim()``` 2. Append ```:*``` at the end of the string to allow to search incomplete string 3. Replace space between multiple words with ```&``` to allow to search incomplete string with multiple words @@ -21,11 +31,44 @@ See important documentation below (according with your postgresql version): - https://www.postgresql.org/docs/current/textsearch.html - https://www.postgresql.org/docs/11/pgtrgm.html - https://stackoverflow.com/questions/2513501/postgresql-full-text-search-how-to-search-partial-words -*/ + +# Examples of Search: SELECT * FROM - + places WHERE - (to_tsvector('spanish',UNACCENT(lower(place_name))) @@ to_tsquery('spanish',UNACCENT(lower(replace(trim(param_string) || ':*',' ','&'))))) or + (to_tsvector('spanish',UNACCENT(lower(name))) @@ to_tsquery('spanish',UNACCENT(lower(replace(trim('ángel') || ':*',' ','&'))))) + + +- First place: "El Salto Ángel" +- Searchs: +ángel +angel +salto +El salto angel +El Salto Ángel +El Sal +salto a +s +sa + sa + +- Second place (with to_tsvector('english'): "The Wizarding World of Harry Potter" +- Searchs: +The Wizarding World of Harry Potter +Harry Potter +Potter +Pot +Potter Harry +w +wi + +- Third place: "Triángulo de las Bermudas" +- Searchs: +triángulo +el triángulo +bermudas + +*/ \ No newline at end of file diff --git a/postgresql/dev_postgres_database/public/tables/places.sql b/postgresql/dev_postgres_database/public/tables/places.sql new file mode 100644 index 0000000..6fe91a6 --- /dev/null +++ b/postgresql/dev_postgres_database/public/tables/places.sql @@ -0,0 +1,8 @@ +CREATE TABLE public.places( + id serial primary key, + name character varying(100) NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + is_active boolean DEFAULT true, + is_deleted boolean DEFAULT false +); \ No newline at end of file From 6c508a4ec434be80d946864cd1b2f5793c48bcd5 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 21 Mar 2020 15:30:19 -0400 Subject: [PATCH 21/24] Updating readme with fts information --- README.md | 2 ++ .../public/functions/useful_fts_queries.sql | 35 ++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 1c537da..b45007b 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ Files are contained inside of the dev_postgres_database folder, with the follow - Find intersection based on WGS 84 - Find near by places within X km from a current position(longitude,latitude) +6. useful_fts_queries.sql: examples of how to do full text search with different case of searches, given stop words, upper or lower cases, accents, white spaces, multiple strings, incomplete strings or multiple incomplete strings. + Contributions ----------------------- diff --git a/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql b/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql index 5e71973..24ff3e4 100644 --- a/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql +++ b/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql @@ -1,32 +1,33 @@ -- To search incoming string in a column - SELECT * FROM WHERE - (to_tsvector('spanish',UNACCENT(lower())) @@ to_tsquery('spanish',UNACCENT(lower(replace(trim() || ':*',' ','&'))))) or (to_tsvector('spanish',UNACCENT(lower())) @@ to_tsquery('spanish',UNACCENT(lower(replace(trim() || ':*',' ','&'))))) - /* # Steps: -1. Remove white spaces at the begining or at the end with ```trim()``` -2. Append ```:*``` at the end of the string to allow to search incomplete string -3. Replace space between multiple words with ```&``` to allow to search incomplete string with multiple words -4. Apply ```lower()``` function -5. Remove accents with ```UNACCENT()```, previously installed with: -``` CREATE EXTENSION unaccent; -``` -6. Remove (spanish) stop words with ```to_tsquery()``` +1. Remove white spaces at the begining or at the end with trim() + +2. Append :* at the end of the string to allow to search incomplete string + +3. Replace space between multiple words with & to allow to search incomplete string with multiple words + +4. Apply lower() function + +5. Remove accents with UNACCENT(), previously installed with: +CREATE EXTENSION unaccent; + +6. Remove (spanish) stop words with to_tsquery() + 7. Match string with the column selected -* Important Note: consider that the column should have some index (GIN, GiST or B-tree, see more information here: https://www.postgresql.org/docs/current/textsearch-indexes.html) to make improvements in fast searchs +* Important Note: consider that the column should have some index (GIN, GiST or B-tree, see more information here: https://www.postgresql.org/docs/current/textsearch-indexes.html) to make improvements in fast searches * To check efficient of this query, use: -``` EXPLAIN ANALYZE query -``` + See important documentation below (according with your postgresql version): - https://www.postgresql.org/docs/current/textsearch.html - https://www.postgresql.org/docs/11/pgtrgm.html @@ -43,7 +44,7 @@ WHERE - First place: "El Salto Ángel" -- Searchs: +- Searches: ángel angel salto @@ -56,7 +57,7 @@ sa sa - Second place (with to_tsvector('english'): "The Wizarding World of Harry Potter" -- Searchs: +- Searches: The Wizarding World of Harry Potter Harry Potter Potter @@ -66,7 +67,7 @@ w wi - Third place: "Triángulo de las Bermudas" -- Searchs: +- Searches: triángulo el triángulo bermudas From b0fb6face98672d9cca468f7912e172d5ec49241 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 21 Mar 2020 18:37:08 -0400 Subject: [PATCH 22/24] Deleting unnecessary lower() function --- .../public/functions/useful_fts_queries.sql | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql b/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql index 24ff3e4..58cb543 100644 --- a/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql +++ b/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql @@ -4,7 +4,7 @@ SELECT FROM WHERE - (to_tsvector('spanish',UNACCENT(lower())) @@ to_tsquery('spanish',UNACCENT(lower(replace(trim() || ':*',' ','&'))))) + (to_tsvector('spanish',UNACCENT()) @@ to_tsquery('spanish',UNACCENT(replace(trim() || ':*',' ','&')))) /* # Steps: @@ -14,14 +14,12 @@ WHERE 3. Replace space between multiple words with & to allow to search incomplete string with multiple words -4. Apply lower() function - -5. Remove accents with UNACCENT(), previously installed with: +4. Remove accents with UNACCENT(), previously installed with: CREATE EXTENSION unaccent; -6. Remove (spanish) stop words with to_tsquery() +5. Remove (spanish) stop words with to_tsquery() -7. Match string with the column selected +6. Match string with the column selected * Important Note: consider that the column should have some index (GIN, GiST or B-tree, see more information here: https://www.postgresql.org/docs/current/textsearch-indexes.html) to make improvements in fast searches @@ -32,6 +30,7 @@ See important documentation below (according with your postgresql version): - https://www.postgresql.org/docs/current/textsearch.html - https://www.postgresql.org/docs/11/pgtrgm.html - https://stackoverflow.com/questions/2513501/postgresql-full-text-search-how-to-search-partial-words +- http://rachbelaid.com/postgres-full-text-search-is-good-enough/ # Examples of Search: @@ -40,7 +39,7 @@ SELECT FROM places WHERE - (to_tsvector('spanish',UNACCENT(lower(name))) @@ to_tsquery('spanish',UNACCENT(lower(replace(trim('ángel') || ':*',' ','&'))))) + (to_tsvector('spanish',UNACCENT(name)) @@ to_tsquery('spanish',UNACCENT(replace(trim('ÁNGEL') || ':*',' ','&')))) - First place: "El Salto Ángel" From b7e6d9aa9280d802c4e73f6c6f7b9d76f40f4e46 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 25 Jul 2020 00:42:00 -0400 Subject: [PATCH 23/24] Adding udf_generate_entity_database_commments function to generate the comment template for tables, columns and functions related with a schema list --- README.md | 2 + ...udf_generate_entity_database_commments.sql | 87 +++++++++++++++++++ .../public/functions/useful_fts_queries.sql | 9 +- 3 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 postgresql/dev_postgres_database/public/functions/udf_generate_entity_database_commments.sql diff --git a/README.md b/README.md index b45007b..4097c4a 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ Files are contained inside of the dev_postgres_database folder, with the follow 4. useful_structure_queries.sql: several useful querys related with schemas, sequences, tables, columns, triggers, constraints, functions and pg_notify channels. +5. udf_generate_entity_database_commments: is a simple script to generate the comment template for tables, columns and functions related with a schema list that you supply. + - List schemas names - List all sequences information - Display all the information about an specific sequence diff --git a/postgresql/dev_postgres_database/public/functions/udf_generate_entity_database_commments.sql b/postgresql/dev_postgres_database/public/functions/udf_generate_entity_database_commments.sql new file mode 100644 index 0000000..76eb77e --- /dev/null +++ b/postgresql/dev_postgres_database/public/functions/udf_generate_entity_database_commments.sql @@ -0,0 +1,87 @@ +-- DROP FUNCTION IF EXISTS public.udf_generate_entity_database_commments(character varying[]); +CREATE OR REPLACE FUNCTION public.udf_generate_entity_database_commments(param_schema_list character varying[]) + RETURNS void + LANGUAGE 'plpgsql' + COST 100 + VOLATILE +AS $BODY$ + +DECLARE + + /* + To test: + SELECT public.udf_generate_entity_database_commments(array['public']); + */ + + i integer; + j RECORD; + k RECORD; + w RECORD; + + BEGIN + + -- Iterate on each schema + FOR i IN 1 .. array_upper(param_schema_list, 1) + LOOP + + -- Iterate on each table of the current schema + FOR j IN ( + SELECT + tablename + FROM + pg_tables + WHERE + schemaname = param_schema_list[i] + ) + LOOP + + -- Generate table comment + RAISE NOTICE 'COMMENT ON TABLE %.% is ''''',param_schema_list[i],j.tablename; + + -- Get all the columns name of the current table + FOR k IN ( + SELECT + column_name + FROM + information_schema.columns + WHERE + table_schema = 'public' + AND + table_name = j.tablename + ORDER BY + column_name + ) + LOOP + + -- Generate column comment + RAISE NOTICE 'COMMENT ON COLUMN %.%.% is ''''',param_schema_list[i],j.tablename,k.column_name; + + END LOOP; + + END LOOP; + + -- Iterate on each function of the current schema + FOR w IN ( + SELECT + proname + FROM + pg_catalog.pg_proc f + INNER JOIN + pg_catalog.pg_namespace n ON (f.pronamespace = n.oid) + WHERE + n.nspname = param_schema_list[i] + ) + LOOP + + -- Generate function comment + RAISE NOTICE 'COMMENT ON FUNCTION %.% is ''''',param_schema_list[i],w.proname; + + END LOOP; + + END LOOP; + +END; + +$BODY$; + +ALTER FUNCTION public.udf_generate_entity_database_commments(character varying[]) OWNER TO postgres; \ No newline at end of file diff --git a/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql b/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql index 58cb543..3e64e9f 100644 --- a/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql +++ b/postgresql/dev_postgres_database/public/functions/useful_fts_queries.sql @@ -1,4 +1,4 @@ --- To search incoming string in a column +-- To search a string in a column SELECT * FROM @@ -21,7 +21,9 @@ CREATE EXTENSION unaccent; 6. Match string with the column selected -* Important Note: consider that the column should have some index (GIN, GiST or B-tree, see more information here: https://www.postgresql.org/docs/current/textsearch-indexes.html) to make improvements in fast searches +* Important Note: consider that the column should have some index (GIN, GiST or B-tree, see more information here: +https://www.postgresql.org/docs/current/textsearch-indexes.html) +To make improvements in fast searches * To check efficient of this query, use: EXPLAIN ANALYZE query @@ -41,12 +43,11 @@ FROM WHERE (to_tsvector('spanish',UNACCENT(name)) @@ to_tsquery('spanish',UNACCENT(replace(trim('ÁNGEL') || ':*',' ','&')))) - - First place: "El Salto Ángel" - Searches: ángel angel -salto +ÁNGEL El salto angel El Salto Ángel El Sal From 21592d7a7f4ded657bf15f732a3518e508dca0a2 Mon Sep 17 00:00:00 2001 From: Manuel Carrero Date: Sat, 25 Jul 2020 17:53:37 -0400 Subject: [PATCH 24/24] Adding query to list information for data dictionary --- README.md | 1 + ...udf_generate_entity_database_commments.sql | 6 ++--- .../functions/useful_structure_queries.sql | 24 +++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4097c4a..99ea367 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ Files are contained inside of the dev_postgres_database folder, with the follow - Search (functions, tables, colums...) in all the functions (Great function!) - List function comments of a schema - List current channels listening +- List information for data dictionary 5. useful_postgis_queries.sql: several useful querys related with geolocation using postgis extension. diff --git a/postgresql/dev_postgres_database/public/functions/udf_generate_entity_database_commments.sql b/postgresql/dev_postgres_database/public/functions/udf_generate_entity_database_commments.sql index 76eb77e..221af87 100644 --- a/postgresql/dev_postgres_database/public/functions/udf_generate_entity_database_commments.sql +++ b/postgresql/dev_postgres_database/public/functions/udf_generate_entity_database_commments.sql @@ -36,7 +36,7 @@ DECLARE LOOP -- Generate table comment - RAISE NOTICE 'COMMENT ON TABLE %.% is ''''',param_schema_list[i],j.tablename; + RAISE NOTICE 'COMMENT ON TABLE %.% is '''';',param_schema_list[i],j.tablename; -- Get all the columns name of the current table FOR k IN ( @@ -54,7 +54,7 @@ DECLARE LOOP -- Generate column comment - RAISE NOTICE 'COMMENT ON COLUMN %.%.% is ''''',param_schema_list[i],j.tablename,k.column_name; + RAISE NOTICE 'COMMENT ON COLUMN %.%.% is '''';',param_schema_list[i],j.tablename,k.column_name; END LOOP; @@ -74,7 +74,7 @@ DECLARE LOOP -- Generate function comment - RAISE NOTICE 'COMMENT ON FUNCTION %.% is ''''',param_schema_list[i],w.proname; + RAISE NOTICE 'COMMENT ON FUNCTION %.% is '''';',param_schema_list[i],w.proname; END LOOP; diff --git a/postgresql/dev_postgres_database/public/functions/useful_structure_queries.sql b/postgresql/dev_postgres_database/public/functions/useful_structure_queries.sql index 8db3f74..00c77f7 100644 --- a/postgresql/dev_postgres_database/public/functions/useful_structure_queries.sql +++ b/postgresql/dev_postgres_database/public/functions/useful_structure_queries.sql @@ -353,3 +353,27 @@ WHERE LIKE '%LISTEN%' ORDER BY backend_start; +----------------------------------------------- +-- RELATED WITH DATA DICTIONARY +----------------------------------------------- + +-- To list information for data dictionary +SELECT + t1.TABLE_SCHEMA, + t1.TABLE_NAME, + t1.COLUMN_NAME, + t1.COLUMN_DEFAULT, + t1.IS_NULLABLE, + t1.DATA_TYPE, + COALESCE(t1.NUMERIC_PRECISION, + t1.CHARACTER_MAXIMUM_LENGTH) AS COLUMN_LENGHT, + PG_CATALOG.COL_DESCRIPTION(t2.OID, + t1.DTD_IDENTIFIER::int) AS COLUMN_DESCRIPTION, + t1.DOMAIN_NAME AS COLUMN_DOMAIN +FROM + INFORMATION_SCHEMA.COLUMNS t1 + INNER JOIN PG_CLASS t2 ON (t2.RELNAME = t1.TABLE_NAME) +WHERE + t1.TABLE_SCHEMA = 'your_schema_name' +ORDER BY + t1.TABLE_NAME;