Open
Show file tree
Hide file tree
Changes from all commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
e423c4e
Alter heuristic dependencies searching mechanism in order to find dep…
rocket-3Jun 11, 2021
1b1aff1
Impl. of fetching aggregate functions on Postgres
rocket-3Jun 13, 2021
b675500
Fix SortedListMetaObject alg., extracted search of name in sql code t…
rocket-3Jun 16, 2021
ea477d8
Fix fetching functions/single function in Postgres
rocket-3Jun 16, 2021
97d26c5
Add IT extension (PathUsingConnectionFromDbLink.java) to operate…
rocket-3Jun 19, 2021
44db0fb
Fix drop constraints order error (now uses _source_ schema objects wh…
rocket-3Jun 19, 2021
96a3bf6
Fix restore domain on Postgres
rocket-3Jun 19, 2021
6518147
Add url parameter on ArgsDbGitLinkPgRemote
rocket-3Jun 19, 2021
1898fcc
Fix restore partition table on Postgres - impl. of dumb table type co…
rocket-3Jun 19, 2021
f87c770
Change DbGitIntegrationTestBasic.dbToDbRestoreWorksWithCustomTypes ba…
rocket-3Jun 19, 2021
cc05f9e
Fix of restore declarative pg table partition
rocket-3Jul 18, 2021
af23edf
Fix of DateData truncates time
rocket-3Jul 18, 2021
ac70cea
Fix of DBAdapterPostgres::getTable(s) on partitioned tables
rocket-3Jul 18, 2021
411757a
Fix/refactor of MetaTableData::loadFromDB for retry loading data only…
rocket-3Jul 18, 2021
428a3ac
Fix PK detection of column in DBAdapterPostgres::getTableFields
rocket-3Jul 21, 2021
06e6594
Refactor IT case 'DbGitIntegrationTestBasic::dbToDbRestoreWorksWithCu…
rocket-3Jul 21, 2021
31fc5b9
IT small refactor
rocket-3Jul 23, 2021
fa21cf1
Fix UB when affected tables for backups come to update objects
rocket-3Jul 23, 2021
ae70f55
Add backups enabled in IT
rocket-3Jul 23, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Failed to load files.
Original file line numberDiff line numberDiff line change
Expand Up@@ -147,6 +147,11 @@

<dependencies>

<dependency>
<groupId>org.cactoos</groupId>
<artifactId>cactoos</artifactId>
<version>0.50</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -109,26 +109,17 @@ public void restoreDataBase(IMapMetaObject updateObjs) throws Exception {

try {
SortedListMetaObject tables = new SortedListMetaObject(updateObjs.values().stream().filter(x->x instanceof MetaTable ).collect(Collectors.toList()));
SortedListMetaObject tablesExists = new SortedListMetaObject(updateObjs.values().stream().filter(x->x instanceof MetaTable && isExists(x)).collect(Collectors.toList()));

Set<String> createdSchemas = getSchemes().values().stream().map(DBOptionsObject::getName).collect(Collectors.toSet());
Set<String> createdRoles = getRoles().values().stream().map(DBRole::getName).collect(Collectors.toSet());

// remove table indexes and constraints, which is step(-2) of restoreMetaObject(MetaTable)
ConsoleWriter.println(lang.getValue("general", "restore", "droppingTablesConstraints"), messageLevel);
for (IMetaObject table : tablesExists.sortFromDependencies()) {
ConsoleWriter.println(lang.getValue("general", "restore", "droppingTableConstraints").withParams(table.getName()), messageLevel+1);
getFactoryRestore().getAdapterRestore(DBGitMetaType.DBGitTable, this).restoreMetaObject(table, -2);
}

for (IMetaObject obj : updateObjs.getSortedList().sortFromReferenced()) {
Timestamp timestampBefore = new Timestamp(System.currentTimeMillis());
int step = 0;
boolean res = false;

IDBAdapterRestoreMetaData restoreAdapter = getFactoryRestore().getAdapterRestore(obj.getType(), this) ;
if(restoreAdapter == null) throw new Exception("restore adapter is null");
// ConsoleWriter.printlnGreen(lang.getValue("general", "restore", "restoreType").withParams(obj.getType().toString().substring(5), obj.getName()));

obj = tryConvert(obj);
createRoleIfNeed(obj, createdRoles);
Expand All@@ -143,11 +134,6 @@ public void restoreDataBase(IMapMetaObject updateObjs) throws Exception {
}

Long timeDiff = new Timestamp(System.currentTimeMillis()).getTime() - timestampBefore.getTime();
// ConsoleWriter.detailsPrintColor(MessageFormat.format(" ({1} {2})"
// , obj.getName()
// , timeDiff
// , lang.getValue("general", "add", "ms")), 0, Ansi.FColor.CYAN
// );
}

// restore table constraints, which is step(-1) of restoreMetaObject(MetaTable)
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,15 +4,22 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.text.MessageFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import java.util.stream.Collectors;

import com.diogonunes.jcdp.color.api.Ansi;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;

import org.cactoos.Scalar;
import org.cactoos.list.ListEnvelope;
import org.cactoos.scalar.ScalarOf;
import org.cactoos.scalar.Sticky;
import ru.fusionsoft.dbgit.adapters.AdapterFactory;
import ru.fusionsoft.dbgit.adapters.IDBAdapter;
import ru.fusionsoft.dbgit.core.*;
Expand DownExpand Up@@ -72,6 +79,7 @@ public void execute(CommandLine cmdLine) throws Exception {
IMapMetaObject fileObjs = gmdm.loadFileMetaData();
IMapMetaObject updateObjs = new TreeMapMetaObject();
IMapMetaObject deleteObjs = new TreeMapMetaObject();
IMapMetaObject backupObjs = new TreeMapMetaObject();

if (toMakeBackup) { ConsoleWriter.printlnColor(getLang().getValue("general", "restore", "willMakeBackup").toString(), Ansi.FColor.GREEN, messageLevel+1); }
else { ConsoleWriter.printlnColor(getLang().getValue("general", "restore", "wontMakeBackup").toString(), Ansi.FColor.GREEN, messageLevel+1); }
Expand DownExpand Up@@ -140,32 +148,84 @@ public void execute(CommandLine cmdLine) throws Exception {

// # steps 1,2 are in GitMetaDataManager::restoreDatabase

ConsoleWriter.println(getLang().getValue("general", "restore", "seekingToRestoreAdditional"),1);
ConsoleWriter.println(getLang().getValue("general", "restore", "seekingToRestoreAdditional"), messageLevel+2);
Map<String, IMetaObject> updateObjectsCopy = new TreeMapMetaObject(updateObjs.values());
Map<String, IMetaObject> affectedTables = new TreeMapMetaObject();
Map<String, IMetaObject> foundTables = new TreeMapMetaObject();
do {
foundTables =
dbObjs.values().stream()
.filter(excluded -> {
return excluded instanceof MetaTable
&& ! updateObjs.containsKey(excluded.getName())
&& updateObjs.values().stream().anyMatch(excluded::dependsOn);
&& ! updateObjectsCopy.containsKey(excluded.getName())
&& updateObjectsCopy.values().stream().anyMatch(excluded::dependsOn);
})
.collect(Collectors.toMap(IMetaObject::getName, val -> val));
affectedTables.putAll(foundTables);
updateObjs.putAll(foundTables);
updateObjectsCopy.putAll(foundTables);
deleteObjs.putAll(foundTables);
backupObjs.putAll(foundTables);
} while (!foundTables.isEmpty());

if(affectedTables.isEmpty()){
ConsoleWriter.println(getLang().getValue("general", "restore", "nothingToRestoreAdditional"), 2);
ConsoleWriter.println(getLang().getValue("general", "restore", "nothingToRestoreAdditional"), messageLevel+2);
} else {
affectedTables.forEach((k,v)->ConsoleWriter.println(k, 2));
affectedTables.forEach((k,v)->ConsoleWriter.println(k, messageLevel+3));
}

//delete MetaSql (but no UDT's, domains or enums) that are in files and in db to fix errors on table restore
ConsoleWriter.println(getLang().getValue("general", "restore", "droppingSqlObjects"), messageLevel+2);
for (final IMetaObject object :
new SortedListMetaObject(
dbObjs.entrySet().stream()
.filter(x -> fileObjs.containsKey(x.getKey()))
.map(Map.Entry::getValue)
.filter(x ->
x instanceof MetaFunction ||
x instanceof MetaProcedure ||
x instanceof MetaView ||
x instanceof MetaTrigger
)
.collect(Collectors.toList())
).sortFromDependencies()
) {
ConsoleWriter.println(
getLang().getValue("general", "restore", "droppingObject").withParams(object.getName()),
messageLevel + 3
);
adapter
.getFactoryRestore()
.getAdapterRestore(object.getType(), adapter)
.removeMetaObject(object);
updateObjs.put(object);
}

// remove table indexes and constraints, which is step(-2) of restoreMetaObject(MetaTable)
ConsoleWriter.println(getLang().getValue("general", "restore", "droppingTablesConstraints"), messageLevel + 2);
for (IMetaObject table : new ScalarOf<List<IMetaObject>>(
ipt -> {
ipt.forEach(x -> ConsoleWriter.println(
MessageFormat.format("{0} ({1})", x.getName(), x.getUnderlyingDbObject().getDependencies()),
messageLevel + 3
));
return ipt;
},
new SortedListMetaObject(
dbObjs.entrySet().stream()
.filter(x -> updateObjs.containsKey(x.getKey()))
.map(Map.Entry::getValue)
.filter(x -> x instanceof MetaTable)
.collect(Collectors.toList())
).sortFromDependencies()
).value()) {
ConsoleWriter.println(
getLang().getValue("general", "restore", "droppingTableConstraints").withParams(table.getName()),
messageLevel + 3
);
adapter.getFactoryRestore().getAdapterRestore(DBGitMetaType.DBGitTable, adapter).restoreMetaObject(table, - 2);
}

if(toMakeBackup && toMakeChanges) {
IMapMetaObject backupObjs = new TreeMapMetaObject();
backupObjs.putAll(deleteObjs);
backupObjs.putAll(updateObjs);
adapter.getBackupAdapterFactory().getBackupAdapter(adapter).backupDatabase(backupObjs);
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
package ru.fusionsoft.dbgit.config;

import java.util.function.Supplier;
import org.cactoos.scalar.Unchecked;
import ru.fusionsoft.dbgit.core.DBGitConfig;

public class TryCount implements Supplier<Integer> {

@Override
public final Integer get() {
return new Unchecked<Integer>(() -> {
return DBGitConfig.getInstance().getInteger(
"core",
"TRY_COUNT",
DBGitConfig.getInstance().getIntegerGlobal("core", "TRY_COUNT", 1000)
);
}).value();
}
}
Original file line numberDiff line numberDiff line change
Expand Up@@ -308,7 +308,7 @@ public IMapMetaObject loadFileMetaData(boolean force) throws ExceptionDBGit {
String filename = files.get(i);
if (DBGitPath.isServiceFile(filename)) continue;

ConsoleWriter.println(DBGitLang.getInstance()
ConsoleWriter.detailsPrintln(DBGitLang.getInstance()
.getValue("general", "meta", "loadFile")
.withParams(filename)
, messageLevel+1
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@

import java.sql.ResultSet;

import java.sql.SQLException;
import ru.fusionsoft.dbgit.core.ExceptionDBGit;
import ru.fusionsoft.dbgit.dbobjects.DBTable;

Expand All@@ -11,7 +12,7 @@ public class BooleanData implements ICellData {
private boolean isNull = false;

@Override
public boolean loadFromDB(ResultSet rs, String fieldName) throws Exception {
public boolean loadFromDB(ResultSet rs, String fieldName) throws SQLException {
value = rs.getBoolean(fieldName);
if (rs.wasNull()) {
isNull = true;
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@

import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;

Expand All@@ -16,12 +17,12 @@ public class DateData implements ICellData {
public static SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmss");

@Override
public boolean loadFromDB(ResultSet rs, String fieldname) throws Exception {
public boolean loadFromDB(ResultSet rs, String fieldname) throws SQLException {
if (rs.getDate(fieldname) == null) {
isNull = true;
value = 0;
} else
value = rs.getDate(fieldname).getTime();
value = rs.getTimestamp(fieldname).getTime();

return true;
}
Expand Down
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
package ru.fusionsoft.dbgit.data_table;

import java.io.IOException;
import java.sql.ResultSet;

import java.sql.SQLException;
import ru.fusionsoft.dbgit.core.ExceptionDBGit;
import ru.fusionsoft.dbgit.dbobjects.DBTable;

public interface ICellData {

public boolean loadFromDB(ResultSet rs, String fieldname) throws Exception;
public boolean loadFromDB(ResultSet rs, String fieldname) throws SQLException, ExceptionDBGit, IOException;

public String serialize(DBTable tbl) throws Exception;

Expand Down
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
package ru.fusionsoft.dbgit.data_table;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Set;

Expand All@@ -25,12 +27,12 @@ public class MapFileData implements ICellData {

//private String hash = null;

public InputStream getBlobData(ResultSet rs, String fieldname) throws Exception {
public InputStream getBlobData(ResultSet rs, String fieldname) throws SQLException, ExceptionDBGit {
return rs.getBinaryStream(fieldname);
}

@Override
public boolean loadFromDB(ResultSet rs, String fieldname) throws Exception {
public boolean loadFromDB(ResultSet rs, String fieldname) throws SQLException, ExceptionDBGit, IOException {
InputStream stream = getBlobData(rs, fieldname);

if (stream != null) {
Expand Down
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
package ru.fusionsoft.dbgit.data_table;

import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.text.MessageFormat;
import java.util.*;

Expand All@@ -25,7 +27,7 @@ public class RowData {
//protected String key;
//protected MetaTable metaTable;

public RowData(ResultSet rs, MetaTable metaTable) throws Exception {
public RowData(ResultSet rs, MetaTable metaTable) throws SQLException, ExceptionDBGit, IOException {
//this.metaTable = metaTable;
loadDataFromRS(rs, metaTable);
}
Expand All@@ -41,7 +43,7 @@ public RowData(CSVRecord record, MetaTable metaTable, CSVRecord titleColumns) th
loadDataFromCSVRecord(record, titleColumns, metaTable);
}

private void loadDataFromRS(ResultSet rs, MetaTable metaTable) throws Exception {
private void loadDataFromRS(ResultSet rs, MetaTable metaTable) throws SQLException, ExceptionDBGit, IOException {
for (int i = 0; i < rs.getMetaData().getColumnCount(); i++) {
String columnName = rs.getMetaData().getColumnName(i+1);

Expand DownExpand Up@@ -142,15 +144,19 @@ public String calcRowKey(List<Integer> idColumns) throws Exception {
}


public String calcRowHash() throws Exception {
CalcHash ch = new CalcHash();
//for (ICellData cd : data.values()) {
for (ICellData cd : rowList) {
String str = cd.convertToString();
if ( str != null)
ch.addData(str);
public String calcRowHash() throws ExceptionDBGit {
try {
CalcHash ch = new CalcHash();
//for (ICellData cd : data.values()) {
for (ICellData cd : rowList) {
String str = cd.convertToString();
if (str != null)
ch.addData(str);
}
return ch.calcHashStr();
} catch (Exception ex){
throw new ExceptionDBGit("Error calculate row hash", ex);
}
return ch.calcHashStr();
}

public Map<String, ICellData> getData(List<String> fields) {
Expand Down
Loading