在利用 hadoop 运行 MapReduce 项目时&＃xff0c;提示报错&＃xff08;注意最后是Z&＃xff09;&＃xff1a;
Exception in thread "main" java.lang.UnsatisfiedLinkError:
org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Ljava/lang/String;I)Z
这个主要是因为本地启动运行&＃xff0c;需要更改源代码中的对应配置。
在项目的根目录下&＃xff08;本项目为WordCount&＃xff0c;根目录也就是java&＃xff09;新建包 org.apache.hadoop.io.nativeio &＃xff0c;添加类 NativeIO 即可。
package org.apache.hadoop.io.nativeio;/*** Licensed to the Apache Software Foundation (ASF) under one* or more contributor license agreements. See the NOTICE file* distributed with this work for additional information* regarding copyright ownership. The ASF licenses this file* to you under the Apache License, Version 2.0 (the* "License"); you may not use this file except in compliance* with the License. You may obtain a copy of the License at** http://www.apache.org/licenses/LICENSE-2.0** Unless required by applicable law or agreed to in writing, software* distributed under the License is distributed on an "AS IS" BASIS,* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.* See the License for the specific language governing permissions and* limitations under the License.*/import java.io.File;
import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.Shell;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;import sun.misc.Unsafe;import com.google.common.annotations.VisibleForTesting;/*** JNI wrappers for various native IO-related calls not available in Java. These* functions should generally be used alongside a fallback to another more* portable mechanism.*/
&＃64;InterfaceAudience.Private
&＃64;InterfaceStability.Unstable
public class NativeIO {public static class POSIX {// Flags for open() call from bits/fcntl.hpublic static final int O_RDONLY &＃61; 00;public static final int O_WRONLY &＃61; 01;public static final int O_RDWR &＃61; 02;public static final int O_CREAT &＃61; 0100;public static final int O_EXCL &＃61; 0200;public static final int O_NOCTTY &＃61; 0400;public static final int O_TRUNC &＃61; 01000;public static final int O_APPEND &＃61; 02000;public static final int O_NONBLOCK &＃61; 04000;public static final int O_SYNC &＃61; 010000;public static final int O_ASYNC &＃61; 020000;public static final int O_FSYNC &＃61; O_SYNC;public static final int O_NDELAY &＃61; O_NONBLOCK;// Flags for posix_fadvise() from bits/fcntl.h/* No further special treatment. */public static final int POSIX_FADV_NORMAL &＃61; 0;/* Expect random page references. */public static final int POSIX_FADV_RANDOM &＃61; 1;/* Expect sequential page references. */public static final int POSIX_FADV_SEQUENTIAL &＃61; 2;/* Will need these pages. */public static final int POSIX_FADV_WILLNEED &＃61; 3;/* Don&＃39;t need these pages. */public static final int POSIX_FADV_DONTNEED &＃61; 4;/* Data will be accessed once. */public static final int POSIX_FADV_NOREUSE &＃61; 5;/** Wait upon writeout of all pages in the range before performing the* write.*/public static final int SYNC_FILE_RANGE_WAIT_BEFORE &＃61; 1;/** Initiate writeout of all those dirty pages in the range which are not* presently under writeback.*/public static final int SYNC_FILE_RANGE_WRITE &＃61; 2;/** Wait upon writeout of all pages in the range after performing the* write.*/public static final int SYNC_FILE_RANGE_WAIT_AFTER &＃61; 4;private static final Log LOG &＃61; LogFactory.getLog(NativeIO.class);private static boolean nativeLoaded &＃61; false;private static boolean fadvisePossible &＃61; true;private static boolean syncFileRangePossible &＃61; true;static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY &＃61; "hadoop.workaround.non.threadsafe.getpwuid";static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT &＃61; true;private static long cacheTimeout &＃61; -1;private static CacheManipulator cacheManipulator &＃61; new CacheManipulator();public static CacheManipulator getCacheManipulator() {return cacheManipulator;}public static void setCacheManipulator(CacheManipulator cacheManipulator) {POSIX.cacheManipulator &＃61; cacheManipulator;}/*** Used to manipulate the operating system cache.*/&＃64;VisibleForTestingpublic static class CacheManipulator {public void mlock(String identifier, ByteBuffer buffer, long len) throws IOException {POSIX.mlock(buffer, len);}public long getMemlockLimit() {return NativeIO.getMemlockLimit();}public long getOperatingSystemPageSize() {return NativeIO.getOperatingSystemPageSize();}public void posixFadviseIfPossible(String identifier, FileDescriptor fd, long offset, long len, int flags)throws NativeIOException {POSIX.posixFadviseIfPossible(identifier, fd, offset, len, flags);}public boolean verifyCanMlock() {return NativeIO.isAvailable();}}/*** A CacheManipulator used for testing which does not actually call* mlock. This allows many tests to be run even when the operating* system does not allow mlock, or only allows limited mlocking.*/&＃64;VisibleForTestingpublic static class NoMlockCacheManipulator extends CacheManipulator {public void mlock(String identifier, ByteBuffer buffer, long len) throws IOException {LOG.info("mlocking " &＃43; identifier);}public long getMemlockLimit() {return 1125899906842624L;}public long getOperatingSystemPageSize() {return 4096;}public boolean verifyCanMlock() {return true;}}static {if (NativeCodeLoader.isNativeCodeLoaded()) {try {Configuration conf &＃61; new Configuration();workaroundNonThreadSafePasswdCalls &＃61; conf.getBoolean(WORKAROUND_NON_THREADSAFE_CALLS_KEY,WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT);initNative();nativeLoaded &＃61; true;cacheTimeout &＃61; conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY,CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT) * 1000;LOG.debug("Initialized cache for IDs to User/Group mapping with a " &＃43; " cache timeout of "&＃43; cacheTimeout / 1000 &＃43; " seconds.");} catch (Throwable t) {// This can happen if the user has an older version of// libhadoop.so// installed - in this case we can continue without native// IO// after warningLOG.error("Unable to initialize NativeIO libraries", t);}}}/*** Return true if the JNI-based native IO extensions are available.*/public static boolean isAvailable() {return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded;}private static void assertCodeLoaded() throws IOException {if (!isAvailable()) {throw new IOException("NativeIO was not loaded");}}/** Wrapper around open(2) */public static native FileDescriptor open(String path, int flags, int mode) throws IOException;/** Wrapper around fstat(2) */private static native Stat fstat(FileDescriptor fd) throws IOException;/*** Native chmod implementation. On UNIX, it is a wrapper around chmod(2)*/private static native void chmodImpl(String path, int mode) throws IOException;public static void chmod(String path, int mode) throws IOException {if (!Shell.WINDOWS) {chmodImpl(path, mode);} else {try {chmodImpl(path, mode);} catch (NativeIOException nioe) {if (nioe.getErrorCode() &＃61;&＃61; 3) {throw new NativeIOException("No such file or directory", Errno.ENOENT);} else {LOG.warn(String.format("NativeIO.chmod error (%d): %s", nioe.getErrorCode(), nioe.getMessage()));throw new NativeIOException("Unknown error", Errno.UNKNOWN);}}}}/** Wrapper around posix_fadvise(2) */static native void posix_fadvise(FileDescriptor fd, long offset, long len, int flags) throws NativeIOException;/** Wrapper around sync_file_range(2) */static native void sync_file_range(FileDescriptor fd, long offset, long nbytes, int flags)throws NativeIOException;/*** Call posix_fadvise on the given file descriptor. See the manpage for* this syscall for more information. On systems where this call is not* available, does nothing.** &＃64;throws NativeIOException* if there is an error with the syscall*/static void posixFadviseIfPossible(String identifier, FileDescriptor fd, long offset, long len, int flags)throws NativeIOException {if (nativeLoaded && fadvisePossible) {try {posix_fadvise(fd, offset, len, flags);} catch (UnsupportedOperationException uoe) {fadvisePossible &＃61; false;} catch (UnsatisfiedLinkError ule) {fadvisePossible &＃61; false;}}}/*** Call sync_file_range on the given file descriptor. See the manpage* for this syscall for more information. On systems where this call is* not available, does nothing.** &＃64;throws NativeIOException* if there is an error with the syscall*/public static void syncFileRangeIfPossible(FileDescriptor fd, long offset, long nbytes, int flags)throws NativeIOException {if (nativeLoaded && syncFileRangePossible) {try {sync_file_range(fd, offset, nbytes, flags);} catch (UnsupportedOperationException uoe) {syncFileRangePossible &＃61; false;} catch (UnsatisfiedLinkError ule) {syncFileRangePossible &＃61; false;}}}static native void mlock_native(ByteBuffer buffer, long len) throws NativeIOException;static native void munlock_native(ByteBuffer buffer, long len) throws NativeIOException;/*** Locks the provided direct ByteBuffer into memory, preventing it from* swapping out. After a buffer is locked, future accesses will not* incur a page fault.** See the mlock(2) man page for more information.** &＃64;throws NativeIOException*/static void mlock(ByteBuffer buffer, long len) throws IOException {assertCodeLoaded();if (!buffer.isDirect()) {throw new IOException("Cannot mlock a non-direct ByteBuffer");}mlock_native(buffer, len);}/*** Unlocks a locked direct ByteBuffer, allowing it to swap out of* memory. This is a no-op if the ByteBuffer was not previously locked.** See the munlock(2) man page for more information.** &＃64;throws NativeIOException*/public static void munlock(ByteBuffer buffer, long len) throws IOException {assertCodeLoaded();if (!buffer.isDirect()) {throw new IOException("Cannot munlock a non-direct ByteBuffer");}munlock_native(buffer, len);}/*** Unmaps the block from memory. See munmap(2).** There isn&＃39;t any portable way to unmap a memory region in Java. So we* use the sun.nio method here. Note that unmapping a memory region* could cause crashes if code continues to reference the unmapped code.* However, if we don&＃39;t manually unmap the memory, we are dependent on* the finalizer to do it, and we have no idea when the finalizer will* run.** &＃64;param buffer* The buffer to unmap.*/public static void munmap(MappedByteBuffer buffer) {if (buffer instanceof sun.nio.ch.DirectBuffer) {sun.misc.Cleaner cleaner &＃61; ((sun.nio.ch.DirectBuffer) buffer).cleaner();cleaner.clean();}}/** Linux only methods used for getOwner() implementation */private static native long getUIDforFDOwnerforOwner(FileDescriptor fd) throws IOException;private static native String getUserName(long uid) throws IOException;/*** Result type of the fstat call*/public static class Stat {private int ownerId, groupId;private String owner, group;private int mode;// Mode constantspublic static final int S_IFMT &＃61; 0170000; /* type of file */public static final int S_IFIFO &＃61; 0010000; /* named pipe (fifo) */public static final int S_IFCHR &＃61; 0020000; /* character special */public static final int S_IFDIR &＃61; 0040000; /* directory */public static final int S_IFBLK &＃61; 0060000; /* block special */public static final int S_IFREG &＃61; 0100000; /* regular */public static final int S_IFLNK &＃61; 0120000; /* symbolic link */public static final int S_IFSOCK &＃61; 0140000; /* socket */public static final int S_IFWHT &＃61; 0160000; /* whiteout */public static final int S_ISUID &＃61; 0004000; /** set user id on* execution*/public static final int S_ISGID &＃61; 0002000; /** set group id on* execution*/public static final int S_ISVTX &＃61; 0001000; /** save swapped text even* after use*/public static final int S_IRUSR &＃61; 0000400; /** read permission, owner*/public static final int S_IWUSR &＃61; 0000200; /** write permission,* owner*/public static final int S_IXUSR &＃61; 0000100; /** execute/search* permission, owner*/Stat(int ownerId, int groupId, int mode) {this.ownerId &＃61; ownerId;this.groupId &＃61; groupId;this.mode &＃61; mode;}Stat(String owner, String group, int mode) {if (!Shell.WINDOWS) {this.owner &＃61; owner;} else {this.owner &＃61; stripDomain(owner);}if (!Shell.WINDOWS) {this.group &＃61; group;} else {this.group &＃61; stripDomain(group);}this.mode &＃61; mode;}&＃64;Overridepublic String toString() {return "Stat(owner&＃61;&＃39;" &＃43; owner &＃43; "&＃39;, group&＃61;&＃39;" &＃43; group &＃43; "&＃39;" &＃43; ", mode&＃61;" &＃43; mode &＃43; ")";}public String getOwner() {return owner;}public String getGroup() {return group;}public int getMode() {return mode;}}/*** Returns the file stat for a file descriptor.** &＃64;param fd* file descriptor.* &＃64;return the file descriptor file stat.* &＃64;throws IOException* thrown if there was an IO error while obtaining the file* stat.*/public static Stat getFstat(FileDescriptor fd) throws IOException {Stat stat &＃61; null;if (!Shell.WINDOWS) {stat &＃61; fstat(fd);stat.owner &＃61; getName(IdCache.USER, stat.ownerId);stat.group &＃61; getName(IdCache.GROUP, stat.groupId);} else {try {stat &＃61; fstat(fd);} catch (NativeIOException nioe) {if (nioe.getErrorCode() &＃61;&＃61; 6) {throw new NativeIOException("The handle is invalid.", Errno.EBADF);} else {LOG.warn(String.format("NativeIO.getFstat error (%d): %s", nioe.getErrorCode(),nioe.getMessage()));throw new NativeIOException("Unknown error", Errno.UNKNOWN);}}}return stat;}private static String getName(IdCache domain, int id) throws IOException {Map
}
京公网安备 11010802041100号 | 京ICP备19059560号-4 | PHP1.CN 第一PHP社区 版权所有 