org.apache.kylin.common.KylinConfig.getHdfsWorkingDirectory()方法的使用及代码示例

本文整理了Java中org.apache.kylin.common.KylinConfig.getHdfsWorkingDirectory()方法的一些代码示例，展示了KylinConfig.getHdfsWorkingDirectory()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。KylinConfig.getHdfsWorkingDirectory()方法的具体详情如下：
包路径：org.apache.kylin.common.KylinConfig
类名称：KylinConfig
方法名：getHdfsWorkingDirectory

KylinConfig.getHdfsWorkingDirectory介绍

暂无

代码示例

代码示例来源：origin: apache/kylin

public String getHdfsWorkingDirectory() {
return config.getHdfsWorkingDirectory();
}

代码示例来源：origin: apache/kylin

@Override
protected String pushdownRootPath() {
String hdfsWorkingDirectory = this.kylinConfig.getHdfsWorkingDirectory(null);
if (hdfsWorkingDirectory.endsWith("/"))
return hdfsWorkingDirectory + "resources";
else
return hdfsWorkingDirectory + "/" + "resources";
}

代码示例来源：origin: apache/kylin

private String convertToAbsolutePath(String path) {
KylinConfig kylinCOnfig= KylinConfig.getInstanceFromEnv();
Path basicPath = new Path(path);
if (basicPath.toUri().getScheme() == null)
return kylinConfig.getHdfsWorkingDirectory() + path;
String[] paths = path.split("/resources/GlobalDict/");
if (paths.length == 2)
return kylinConfig.getHdfsWorkingDirectory() + "/resources/GlobalDict/" + paths[1];
paths = path.split("/resources/SegmentDict/");
if (paths.length == 2) {
return kylinConfig.getHdfsWorkingDirectory() + "/resources/SegmentDict/" + paths[1];
} else {
throw new RuntimeException(
"the basic directory of global dictionary only support the format which contains '/resources/GlobalDict/' or '/resources/SegmentDict/'");
}
}

代码示例来源：origin: apache/kylin

private static Path getCoprocessorHDFSDir(FileSystem fileSystem, KylinConfig config) throws IOException {
String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
hdfsWorkingDirectory = HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory);
Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
fileSystem.mkdirs(coprocessorDir);
return coprocessorDir;
}

代码示例来源：origin: apache/kylin

/**
* JIRA: https://issues.apache.org/jira/browse/KYLIN-2945
* if pass a absolute path, it may produce some problems like cannot find global dict after migration.
* so convert to relative path can avoid it and be better to maintain flexibility.
*
*/
private String convertToRelativePath(String path) {
KylinConfig kylinCOnfig= KylinConfig.getInstanceFromEnv();
String hdfsWorkingDir = kylinConfig.getHdfsWorkingDirectory();
if (!isSaveAbsolutePath && path.startsWith(hdfsWorkingDir)) {
return path.substring(hdfsWorkingDir.length());
}
return path;
}

代码示例来源：origin: apache/kylin

@Override
public String copyToAnotherMeta(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
if (baseDir.contains("resources/SegmentDict")) {
logger.info("SegmentAppendTrieDict needn't to copy");
return baseDir;
}
checkArgument(baseDir.startsWith(srcConfig.getHdfsWorkingDirectory()),
"Please check why current directory {} doesn't belong to source working directory {}", baseDir,
srcConfig.getHdfsWorkingDirectory());
final String dstBaseDir = baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(),
dstConfig.getHdfsWorkingDirectory());
Long[] versiOns= listAllVersions();
if (versions.length == 0) { // empty dict, nothing to copy
return dstBaseDir;
}
Path srcVersiOnDir= getVersionDir(versions[versions.length - 1]);
Path dstVersiOnDir= new Path(srcVersionDir.toString().replaceFirst(srcConfig.getHdfsWorkingDirectory(),
dstConfig.getHdfsWorkingDirectory()));
FileSystem dstFS = dstVersionDir.getFileSystem(conf);
if (dstFS.exists(dstVersionDir)) {
dstFS.delete(dstVersionDir, true);
}
FileUtil.copy(fileSystem, srcVersionDir, dstFS, dstVersionDir, false, true, conf);
return dstBaseDir;
}

代码示例来源：origin: apache/kylin

public static FileSystem getWorkingFileSystem(Configuration conf) throws IOException {
Path workingPath = new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
return getFileSystem(workingPath, conf);
}

代码示例来源：origin: apache/kylin

public BaseBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
KylinConfig cOnfig= KylinConfig.getInstanceFromEnv();
this.flatDesc = flatDesc;
this.flatTableDatabase = config.getHiveDatabaseForIntermediateTable();
this.hdfsWorkingDir = config.getHdfsWorkingDirectory();
}

代码示例来源：origin: apache/kylin

public static void main(String[] args) throws IOException {
String path = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict/";
if (args.length > 0) {
path = args[0];
}
System.out.println("Recursive Check AppendTrieDictionary Slices in path " + path);
AppendTrieDictionaryChecker checker = new AppendTrieDictionaryChecker();
if (checker.runChecker(path)) {
System.exit(0);
} else {
System.exit(-1);
}
}
}

代码示例来源：origin: apache/kylin

public static FileSystem getWorkingFileSystem() throws IOException {
return getFileSystem(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
}

代码示例来源：origin: apache/kylin

protected String getJobWorkingDir(DefaultChainedExecutable jobFlow) {
return JobBuilderSupport.getJobWorkingDir(config.getHdfsWorkingDirectory(), jobFlow.getId());
}

代码示例来源：origin: apache/kylin

private static void renameFoldersInHdfs(CubeInstance cube) {
for (CubeSegment segment : cube.getSegments()) {
String jobUuid = segment.getLastBuildJobID();
String src = JobBuilderSupport.getJobWorkingDir(srcConfig.getHdfsWorkingDirectory(), jobUuid);
String tgt = JobBuilderSupport.getJobWorkingDir(dstConfig.getHdfsWorkingDirectory(), jobUuid);
operations.add(new Opt(OptType.RENAME_FOLDER_IN_HDFS, new Object[] { src, tgt }));
}
}

代码示例来源：origin: apache/kylin

protected void renameFoldersInHdfs(CubeInstance cube) throws IOException {
for (CubeSegment segment : cube.getSegments()) {
String jobUuid = segment.getLastBuildJobID();
String src = JobBuilderSupport.getJobWorkingDir(srcConfig.getHdfsWorkingDirectory(), jobUuid);
String tgt = JobBuilderSupport.getJobWorkingDir(dstConfig.getHdfsWorkingDirectory(), jobUuid);
operations.add(new Opt(OptType.RENAME_FOLDER_IN_HDFS, new Object[] { src, tgt }));
}
}

代码示例来源：origin: apache/kylin

public HDFSResourceStore(KylinConfig kylinConfig, StorageURL metadataUrl) throws Exception {
super(kylinConfig);
Preconditions.checkState(HDFS_SCHEME.equals(metadataUrl.getScheme()));
String path = metadataUrl.getParameter("path");
if (path == null) {
// missing path is not expected, but don't fail it
path = kylinConfig.getHdfsWorkingDirectory(null) + "tmp_metadata";
logger.warn("Missing path, fall back to {}. ", path);
}
fs = HadoopUtil.getFileSystem(path);
Path metadataPath = new Path(path);
if (fs.exists(metadataPath) == false) {
logger.warn("Path not exist in HDFS, create it: {}. ", path);
createMetaFolder(metadataPath);
}
hdfsMetaPath = metadataPath;
logger.info("hdfs meta path : {}", hdfsMetaPath);
}

代码示例来源：origin: apache/kylin

protected String getLocalWorkingDirectory() {
String dir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
if (dir.startsWith("file://"))
dir = dir.substring("file://".length());
try {
return new File(dir).getCanonicalPath();
} catch (IOException e) {
throw new RuntimeException(e);
}
}

代码示例来源：origin: apache/kylin

@Override
public void init(DictionaryInfo dictInfo, int baseId, String hdfsDir) throws IOException {
sourceColumn = dictInfo.getSourceTable() + "." + dictInfo.getSourceColumn();
KylinConfig cOnfig= KylinConfig.getInstanceFromEnv();
int maxEntriesPerSlice = config.getAppendDictEntrySize();
if (hdfsDir == null) {
//build in Kylin job server
hdfsDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
}
//use UUID to make each segment dict in different HDFS dir and support concurrent build
//use timestamp to make the segment dict easily to delete
String baseDir = hdfsDir + "resources/SegmentDict" + dictInfo.getResourceDir() + "/"
+ RandomUtil.randomUUID().toString() + "_" + System.currentTimeMillis() + "/";
this.builder = new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, false);
this.baseId = baseId;
}

代码示例来源：origin: apache/kylin

@Test
public void testHdfsWorkingDir() {
KylinConfig cOnf= KylinConfig.getInstanceFromEnv();
String hdfsWorkingDirectory = conf.getHdfsWorkingDirectory();
assertTrue(hdfsWorkingDirectory.startsWith("file:/"));
}

代码示例来源：origin: apache/kylin

@Test
public void TestGetJobWorkingDir() throws IOException {
FileSystem fileSystem = FileSystem.get(new Configuration());
Path jobWorkDirPath = null;
KylinConfig kylinCOnfig= mock(KylinConfig.class);
try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(kylinConfig)) {
when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true);
when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/");
DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class);
defaultChainedExecutable.setId(RandomUtil.randomUUID().toString());
String jobWorkingDir = HiveInputBase.getJobWorkingDir(defaultChainedExecutable,
KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
jobWorkDirPath = new Path(jobWorkingDir);
Assert.assertTrue(fileSystem.exists(jobWorkDirPath));
} finally {
if (jobWorkDirPath != null)
fileSystem.deleteOnExit(jobWorkDirPath);
}
}

代码示例来源：origin: apache/kylin

@Override
public void init(DictionaryInfo dictInfo, int baseId, String hdfsDir) throws IOException {
sourceColumn = dictInfo.getSourceTable() + "_" + dictInfo.getSourceColumn();
lock = KylinConfig.getInstanceFromEnv().getDistributedLockFactory().lockForCurrentThread();
lock.lock(getLockPath(sourceColumn), Long.MAX_VALUE);
int maxEntriesPerSlice = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize();
if (hdfsDir == null) {
//build in Kylin job server
hdfsDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
}
String baseDir = hdfsDir + "resources/GlobalDict" + dictInfo.getResourceDir() + "/";
try {
this.builder = new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, true);
} catch (Throwable e) {
lock.unlock(getLockPath(sourceColumn));
throw new RuntimeException(
String.format(Locale.ROOT, "Failed to create global dictionary on %s ", sourceColumn), e);
}
this.baseId = baseId;
}

代码示例来源：origin: apache/kylin

@Before
public void beforeTest() {
staticCreateTestMetadata();
KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "50000");
BASE_DIR = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR
+ "/";
LOCAL_BASE_DIR = getLocalWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR + "/";
}